diff --git a/configure.ac b/configure.ac index aed27d36d5..53c430ff45 100644 --- a/configure.ac +++ b/configure.ac @@ -720,7 +720,6 @@ then # this set is also enabled by enable-all-crypto: test "$enable_atomicuser" = "" && enable_atomicuser=yes test "$enable_aesgcm" = "" && enable_aesgcm=yes - test "$enable_aesgcm_stream" = "" && test "$enable_aesgcm" = "yes" && enable_aesgcm_stream=yes test "$enable_aesccm" = "" && enable_aesccm=yes test "$enable_aesctr" = "" && enable_aesctr=yes test "$enable_aeseax" = "" && enable_aeseax=yes @@ -741,22 +740,22 @@ then test "$enable_hkdf" = "" && enable_hkdf=yes test "$enable_curve25519" = "" && enable_curve25519=yes test "$enable_curve448" = "" && enable_curve448=yes - test "$enable_fpecc" = "" && enable_fpecc=yes - test "$enable_eccencrypt" = "" && enable_eccencrypt=yes + test "$enable_fpecc" = "" && test "$enable_ecc" != "no" && enable_fpecc=yes + test "$enable_eccencrypt" = "" && test "$enable_ecc" != "no" && enable_eccencrypt=yes test "$enable_psk" = "" && enable_psk=yes test "$enable_cmac" = "" && enable_cmac=yes test "$enable_siphash" = "" && enable_siphash=yes test "$enable_xts" = "" && enable_xts=yes test "$enable_ocsp" = "" && enable_ocsp=yes - test "$enable_ocspstapling" = "" && enable_ocspstapling=yes - test "$enable_ocspstapling2" = "" && enable_ocspstapling2=yes + test "$enable_ocspstapling" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling=yes + test "$enable_ocspstapling2" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling2=yes test "$enable_crl" = "" && enable_crl=yes test "$enable_supportedcurves" = "" && enable_supportedcurves=yes test "$enable_tlsx" = "" && enable_tlsx=yes test "$enable_pwdbased" = "" && enable_pwdbased=yes test "$enable_aeskeywrap" = "" && enable_aeskeywrap=yes test "$enable_x963kdf" = "" && enable_x963kdf=yes - test "$enable_scrypt" = "" && enable_scrypt=yes + test "$enable_scrypt" = "" && test "$enable_hmac" != "no" && enable_scrypt=yes test "$enable_indef" = "" && enable_indef=yes test "$enable_enckeys" = "" && enable_enckeys=yes test "$enable_hashflags" = "" && enable_hashflags=yes @@ -771,7 +770,7 @@ then test "$enable_md4" = "" && enable_md4=yes test "$enable_cryptocb" = "" && enable_cryptocb=yes test "$enable_anon" = "" && enable_anon=yes - test "$enable_ssh" = "" && enable_ssh=yes + test "$enable_ssh" = "" && test "$enable_hmac" != "no" && enable_ssh=yes test "$enable_savesession" = "" && enable_savesession=yes test "$enable_savecert" = "" && enable_savecert=yes @@ -797,6 +796,7 @@ then if test "$ENABLED_LINUXKM_DEFAULTS" != "yes" then + test "$enable_aesgcm_stream" = "" && test "$enable_aesgcm" = "yes" && enable_aesgcm_stream=yes test "$enable_compkey" = "" && enable_compkey=yes test "$enable_quic" = "" && enable_quic=yes AM_CFLAGS="$AM_CFLAGS -DHAVE_CRL_IO -DHAVE_IO_TIMEOUT" @@ -836,7 +836,7 @@ then if test "$ENABLED_FIPS" = "no" then test "$enable_pkcallbacks" = "" && enable_pkcallbacks=yes - test "$enable_xchacha" = "" && enable_xchacha=yes + test "$enable_xchacha" = "" && test "$enable_chacha" != "no" && enable_xchacha=yes test "$enable_scep" = "" && enable_scep=yes test "$enable_pkcs7" = "" && enable_pkcs7=yes test "$enable_nullcipher" = "" && enable_nullcipher=yes @@ -844,9 +844,9 @@ then if test "$ENABLED_32BIT" != "yes" then test "$enable_ed25519" = "" && enable_ed25519=yes - test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes + test "$enable_ed25519_stream" = "" && test "$enable_ed25519" != "no" && enable_ed25519_stream=yes test "$enable_ed448" = "" && enable_ed448=yes - test "$enable_ed448_stream" = "" && enable_ed448_stream=yes + test "$enable_ed448_stream" = "" && test "$enable_ed448" != "no" && enable_ed448_stream=yes fi if test "$ENABLED_LINUXKM_DEFAULTS" != "yes" @@ -856,8 +856,8 @@ then test "$enable_curl" = "" && enable_curl=yes test "$enable_tcpdump" = "" && enable_tcpdump=yes - test "$enable_eccsi" = "" && enable_eccsi=yes - test "$enable_sakke" = "" && enable_sakke=yes + test "$enable_eccsi" = "" && test "$enable_ecc" != "no" && enable_eccsi=yes + test "$enable_sakke" = "" && test "$enable_ecc" != "no" && enable_sakke=yes fi fi @@ -908,7 +908,6 @@ if test "$ENABLED_ALL_CRYPT" = "yes" then test "$enable_atomicuser" = "" && enable_atomicuser=yes test "$enable_aesgcm" = "" && enable_aesgcm=yes - test "$enable_aesgcm_stream" = "" && test "$enable_aesgcm" = "yes" && enable_aesgcm_stream=yes test "$enable_aesccm" = "" && enable_aesccm=yes test "$enable_aesctr" = "" && enable_aesctr=yes test "$enable_aeseax" = "" && enable_aeseax=yes @@ -929,22 +928,22 @@ then test "$enable_hkdf" = "" && enable_hkdf=yes test "$enable_curve25519" = "" && enable_curve25519=yes test "$enable_curve448" = "" && enable_curve448=yes - test "$enable_fpecc" = "" && enable_fpecc=yes - test "$enable_eccencrypt" = "" && enable_eccencrypt=yes + test "$enable_fpecc" = "" && test "$enable_ecc" != "no" && enable_fpecc=yes + test "$enable_eccencrypt" = "" && test "$enable_ecc" != "no" && enable_eccencrypt=yes test "$enable_psk" = "" && enable_psk=yes test "$enable_cmac" = "" && enable_cmac=yes test "$enable_siphash" = "" && enable_siphash=yes test "$enable_xts" = "" && enable_xts=yes test "$enable_ocsp" = "" && enable_ocsp=yes - test "$enable_ocspstapling" = "" && enable_ocspstapling=yes - test "$enable_ocspstapling2" = "" && enable_ocspstapling2=yes + test "$enable_ocspstapling" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling=yes + test "$enable_ocspstapling2" = "" && test "$enable_ocsp" != "no" && enable_ocspstapling2=yes test "$enable_crl" = "" && enable_crl=yes test "$enable_supportedcurves" = "" && enable_supportedcurves=yes test "$enable_tlsx" = "" && enable_tlsx=yes test "$enable_pwdbased" = "" && enable_pwdbased=yes test "$enable_aeskeywrap" = "" && enable_aeskeywrap=yes test "$enable_x963kdf" = "" && enable_x963kdf=yes - test "$enable_scrypt" = "" && enable_scrypt=yes + test "$enable_scrypt" = "" && test "$enable_hmac" != "no" && enable_scrypt=yes test "$enable_indef" = "" && enable_indef=yes test "$enable_enckeys" = "" && enable_enckeys=yes test "$enable_hashflags" = "" && enable_hashflags=yes @@ -959,7 +958,7 @@ then test "$enable_md4" = "" && enable_md4=yes test "$enable_cryptocb" = "" && enable_cryptocb=yes test "$enable_anon" = "" && enable_anon=yes - test "$enable_ssh" = "" && enable_ssh=yes + test "$enable_ssh" = "" && test "$enable_hmac" != "no" && enable_ssh=yes if test "$ENABLED_32BIT" != "yes" then @@ -969,6 +968,7 @@ then if test "$ENABLED_LINUXKM_DEFAULTS" != "yes" then + test "$enable_aesgcm_stream" = "" && test "$enable_aesgcm" = "yes" && enable_aesgcm_stream=yes test "$enable_compkey" = "" && enable_compkey=yes fi @@ -983,21 +983,21 @@ then if test "$ENABLED_FIPS" = "no" then test "$enable_pkcallbacks" = "" && enable_pkcallbacks=yes - test "$enable_xchacha" = "" && enable_xchacha=yes + test "$enable_xchacha" = "" && test "$enable_chacha" != "no" && enable_xchacha=yes test "$enable_pkcs7" = "" && enable_pkcs7=yes test "$enable_nullcipher" = "" && enable_nullcipher=yes if test "$ENABLED_32BIT" != "yes" then test "$enable_ed25519" = "" && enable_ed25519=yes - test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes + test "$enable_ed25519_stream" = "" && test "$enable_ed25519" != "no" && enable_ed25519_stream=yes test "$enable_ed448" = "" && enable_ed448=yes - test "$enable_ed448_stream" = "" && enable_ed448_stream=yes + test "$enable_ed448_stream" = "" && test "$enable_ed448" != "no" && enable_ed448_stream=yes fi if test "$ENABLED_LINUXKM_DEFAULTS" != "yes" then - test "$enable_eccsi" = "" && enable_eccsi=yes - test "$enable_sakke" = "" && enable_sakke=yes + test "$enable_eccsi" = "" && test "$enable_ecc" != "no" && enable_eccsi=yes + test "$enable_sakke" = "" && test "$enable_ecc" != "no" && enable_sakke=yes fi fi @@ -2896,6 +2896,10 @@ then if test "$ENABLED_AESNI" = "yes" || test "$ENABLED_INTELASM" = "yes" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AESNI" + if test "$ENABLED_LINUXKM_DEFAULTS" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DWC_AES_C_DYNAMIC_FALLBACK" + fi if test "$CC" != "icc" then case $host_os in @@ -3951,6 +3955,10 @@ AC_ARG_ENABLE([eccsi], if test "x$ENABLED_ECCSI" = "xyes" then + if test "$ENABLED_ECC" = "no" + then + AC_MSG_ERROR([ECCSI requires ECC.]) + fi AM_CFLAGS="$AM_CFLAGS -DWOLFCRYPT_HAVE_ECCSI -DWOLFSSL_PUBLIC_MP" fi @@ -3961,6 +3969,11 @@ AC_ARG_ENABLE([sakke], [ ENABLED_SAKKE=no ] ) +if test "$ENABLED_SAKKE" != "no" && test "$ENABLED_ECC" = "no" +then + AC_MSG_ERROR([SAKKE requires ECC.]) +fi + if test "x$ENABLED_SAKKE" = "xsmall" then ENABLED_SAKKE="yes" @@ -8926,6 +8939,11 @@ if test "x$ENABLED_OPENSSLCOEXIST" = "xyes"; then fi fi +if test "$ENABLED_WOLFSSH" = "yes" && test "$ENABLED_HMAC" = "no" +then + AC_MSG_ERROR([WOLFSSH requires HMAC.]) +fi + AS_IF([test "x$ENABLED_WOLFSSH" = "xyes"],[AM_CPPFLAGS="$AM_CPPFLAGS -DWOLFSSL_WOLFSSH"]) # only allow secure renegotiation info with TLSV12 and ASN diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 31ca27bf3e..3e738d00b1 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -185,6 +185,7 @@ #endif #ifndef SAVE_VECTOR_REGISTERS #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_x86(); if (_svr_ret != 0) { fail_clause } } + #define SAVE_VECTOR_REGISTERS2() save_vector_registers_x86() #endif #ifndef RESTORE_VECTOR_REGISTERS #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86() @@ -193,6 +194,7 @@ #include #ifndef SAVE_VECTOR_REGISTERS #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } } + #define SAVE_VECTOR_REGISTERS2() save_vector_registers_arm() #endif #ifndef RESTORE_VECTOR_REGISTERS #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm() diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 1239909d34..2fad0eb447 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -2125,7 +2125,9 @@ static void bench_stats_sym_finish(const char* desc, int useDeviceID, (void)useDeviceID; (void)ret; +#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS RESTORE_VECTOR_REGISTERS(); +#endif TEST_SLEEP(); } /* bench_stats_sym_finish */ @@ -2283,7 +2285,9 @@ static void bench_stats_asym_finish_ex(const char* algo, int strength, (void)useDeviceID; (void)ret; +#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS RESTORE_VECTOR_REGISTERS(); +#endif TEST_SLEEP(); } /* bench_stats_asym_finish_ex */ diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 1fc0b003fa..beecef5ad4 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -535,7 +535,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits #define AESNI_ALIGN 16 #endif - static int checkAESNI = 0; + static int checkedAESNI = 0; static int haveAESNI = 0; static word32 intel_flags = 0; @@ -550,71 +550,73 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits /* tell C compiler these are asm functions in case any mix up of ABI underscore prefix between clang/gcc/llvm etc */ #ifdef HAVE_AES_CBC - void AES_CBC_encrypt(const unsigned char* in, unsigned char* out, + void AES_CBC_encrypt_AESNI(const unsigned char* in, unsigned char* out, unsigned char* ivec, unsigned long length, const unsigned char* KS, int nr) - XASM_LINK("AES_CBC_encrypt"); + XASM_LINK("AES_CBC_encrypt_AESNI"); #ifdef HAVE_AES_DECRYPT #if defined(WOLFSSL_AESNI_BY4) || defined(WOLFSSL_X86_BUILD) - void AES_CBC_decrypt_by4(const unsigned char* in, unsigned char* out, + void AES_CBC_decrypt_AESNI_by4(const unsigned char* in, unsigned char* out, unsigned char* ivec, unsigned long length, const unsigned char* KS, int nr) - XASM_LINK("AES_CBC_decrypt_by4"); + XASM_LINK("AES_CBC_decrypt_AESNI_by4"); #elif defined(WOLFSSL_AESNI_BY6) - void AES_CBC_decrypt_by6(const unsigned char* in, unsigned char* out, + void AES_CBC_decrypt_AESNI_by6(const unsigned char* in, unsigned char* out, unsigned char* ivec, unsigned long length, const unsigned char* KS, int nr) - XASM_LINK("AES_CBC_decrypt_by6"); + XASM_LINK("AES_CBC_decrypt_AESNI_by6"); #else /* WOLFSSL_AESNI_BYx */ - void AES_CBC_decrypt_by8(const unsigned char* in, unsigned char* out, + void AES_CBC_decrypt_AESNI_by8(const unsigned char* in, unsigned char* out, unsigned char* ivec, unsigned long length, const unsigned char* KS, int nr) - XASM_LINK("AES_CBC_decrypt_by8"); + XASM_LINK("AES_CBC_decrypt_AESNI_by8"); #endif /* WOLFSSL_AESNI_BYx */ #endif /* HAVE_AES_DECRYPT */ #endif /* HAVE_AES_CBC */ - void AES_ECB_encrypt(const unsigned char* in, unsigned char* out, + void AES_ECB_encrypt_AESNI(const unsigned char* in, unsigned char* out, unsigned long length, const unsigned char* KS, int nr) - XASM_LINK("AES_ECB_encrypt"); + XASM_LINK("AES_ECB_encrypt_AESNI"); #ifdef HAVE_AES_DECRYPT - void AES_ECB_decrypt(const unsigned char* in, unsigned char* out, + void AES_ECB_decrypt_AESNI(const unsigned char* in, unsigned char* out, unsigned long length, const unsigned char* KS, int nr) - XASM_LINK("AES_ECB_decrypt"); + XASM_LINK("AES_ECB_decrypt_AESNI"); #endif - void AES_128_Key_Expansion(const unsigned char* userkey, + void AES_128_Key_Expansion_AESNI(const unsigned char* userkey, unsigned char* key_schedule) - XASM_LINK("AES_128_Key_Expansion"); + XASM_LINK("AES_128_Key_Expansion_AESNI"); - void AES_192_Key_Expansion(const unsigned char* userkey, + void AES_192_Key_Expansion_AESNI(const unsigned char* userkey, unsigned char* key_schedule) - XASM_LINK("AES_192_Key_Expansion"); + XASM_LINK("AES_192_Key_Expansion_AESNI"); - void AES_256_Key_Expansion(const unsigned char* userkey, + void AES_256_Key_Expansion_AESNI(const unsigned char* userkey, unsigned char* key_schedule) - XASM_LINK("AES_256_Key_Expansion"); + XASM_LINK("AES_256_Key_Expansion_AESNI"); - static WARN_UNUSED_RESULT int AES_set_encrypt_key( + static WARN_UNUSED_RESULT int AES_set_encrypt_key_AESNI( const unsigned char *userKey, const int bits, Aes* aes) { int ret; + ASSERT_SAVED_VECTOR_REGISTERS(); + if (!userKey || !aes) return BAD_FUNC_ARG; switch (bits) { case 128: - AES_128_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 10; + AES_128_Key_Expansion_AESNI (userKey,(byte*)aes->key); aes->rounds = 10; return 0; case 192: - AES_192_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 12; + AES_192_Key_Expansion_AESNI (userKey,(byte*)aes->key); aes->rounds = 12; return 0; case 256: - AES_256_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 14; + AES_256_Key_Expansion_AESNI (userKey,(byte*)aes->key); aes->rounds = 14; return 0; default: ret = BAD_FUNC_ARG; @@ -624,7 +626,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits } #ifdef HAVE_AES_DECRYPT - static WARN_UNUSED_RESULT int AES_set_decrypt_key( + static WARN_UNUSED_RESULT int AES_set_decrypt_key_AESNI( const unsigned char* userKey, const int bits, Aes* aes) { word32 nr; @@ -636,6 +638,8 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits __m128i *Key_Schedule; __m128i *Temp_Key_Schedule; + ASSERT_SAVED_VECTOR_REGISTERS(); + if (!userKey || !aes) return BAD_FUNC_ARG; @@ -645,7 +649,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits return MEMORY_E; #endif - if (AES_set_encrypt_key(userKey,bits,temp_key) == BAD_FUNC_ARG) { + if (AES_set_encrypt_key_AESNI(userKey,bits,temp_key) == BAD_FUNC_ARG) { #ifdef WOLFSSL_SMALL_STACK XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES); #endif @@ -658,12 +662,6 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits nr = temp_key->rounds; aes->rounds = nr; -#ifdef WOLFSSL_SMALL_STACK - SAVE_VECTOR_REGISTERS(XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES); return _svr_ret;); -#else - SAVE_VECTOR_REGISTERS(return _svr_ret;); -#endif - Key_Schedule[nr] = Temp_Key_Schedule[0]; Key_Schedule[nr-1] = _mm_aesimc_si128(Temp_Key_Schedule[1]); Key_Schedule[nr-2] = _mm_aesimc_si128(Temp_Key_Schedule[2]); @@ -687,8 +685,6 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits Key_Schedule[0] = Temp_Key_Schedule[nr]; - RESTORE_VECTOR_REGISTERS(); - #ifdef WOLFSSL_SMALL_STACK XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES); #endif @@ -1830,7 +1826,11 @@ static void AesEncrypt_C(Aes* aes, const byte* inBlock, byte* outBlock, word32 t0, t1, t2, t3; const word32* rk; +#ifdef WC_AES_C_DYNAMIC_FALLBACK + rk = aes->key_C_fallback; +#else rk = aes->key; +#endif /* * map byte array block to cipher state @@ -2680,7 +2680,9 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt( } #ifdef WOLFSSL_AESNI - if (haveAESNI && aes->use_aesni) { + if (aes->use_aesni) { + ASSERT_SAVED_VECTOR_REGISTERS(); + #ifdef DEBUG_AESNI printf("about to aes encrypt\n"); printf("in = %p\n", inBlock); @@ -2702,7 +2704,7 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt( tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN)); XMEMCPY(tmp_align, inBlock, AES_BLOCK_SIZE); - AES_ECB_encrypt(tmp_align, tmp_align, AES_BLOCK_SIZE, + AES_ECB_encrypt_AESNI(tmp_align, tmp_align, AES_BLOCK_SIZE, (byte*)aes->key, (int)aes->rounds); XMEMCPY(outBlock, tmp_align, AES_BLOCK_SIZE); XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER); @@ -2714,7 +2716,7 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt( #endif } - AES_ECB_encrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key, + AES_ECB_encrypt_AESNI(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key, (int)aes->rounds); return 0; @@ -2724,7 +2726,7 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt( printf("Skipping AES-NI\n"); #endif } -#endif +#endif /* WOLFSSL_AESNI */ #if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES) AES_ECB_encrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE); return 0; @@ -2820,7 +2822,11 @@ static void AesDecrypt_C(Aes* aes, const byte* inBlock, byte* outBlock, word32 t0, t1, t2, t3; const word32* rk; +#ifdef WC_AES_C_DYNAMIC_FALLBACK + rk = aes->key_C_fallback; +#else rk = aes->key; +#endif /* * map byte array block to cipher state @@ -3416,7 +3422,9 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt( } #ifdef WOLFSSL_AESNI - if (haveAESNI && aes->use_aesni) { + if (aes->use_aesni) { + ASSERT_SAVED_VECTOR_REGISTERS(); + #ifdef DEBUG_AESNI printf("about to aes decrypt\n"); printf("in = %p\n", inBlock); @@ -3429,7 +3437,7 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt( /* if input and output same will overwrite input iv */ if ((const byte*)aes->tmp != inBlock) XMEMCPY(aes->tmp, inBlock, AES_BLOCK_SIZE); - AES_ECB_decrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key, + AES_ECB_decrypt_AESNI(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key, (int)aes->rounds); return 0; } @@ -3855,6 +3863,7 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt( * or perhaps there's HW support for *some* keylengths * and we need both HW and SW. */ #ifdef NEED_SOFTWARE_AES_SETKEY + #ifndef WC_AES_BITSLICED /* Set the AES key and expand. * @@ -3865,7 +3874,11 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt( */ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) { +#ifdef WC_AES_C_DYNAMIC_FALLBACK + word32* rk = aes->key_C_fallback; +#else word32* rk = aes->key; +#endif word32 temp; unsigned int i = 0; @@ -4000,7 +4013,12 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #if defined(HAVE_AES_DECRYPT) if (dir == AES_DECRYPTION) { unsigned int j; + +#ifdef WC_AES_C_DYNAMIC_FALLBACK + rk = aes->key_C_fallback; +#else rk = aes->key; +#endif /* invert the order of the round keys: */ for (i = 0, j = 4* aes->rounds; i < j; i += 4, j -= 4) { @@ -4041,13 +4059,14 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #else (void)dir; #endif /* HAVE_AES_DECRYPT */ - (void)temp; #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(&temp, sizeof(temp)); +#else + (void)temp; #endif } -#else +#else /* WC_AES_BITSLICED */ /* Set the AES key and expand. * * @param [in] aes AES object. @@ -4062,7 +4081,7 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) bs_set_key(aes->bs_key, key, keySz, aes->rounds); } -#endif /* !WC_AES_BITSLICED */ +#endif /* WC_AES_BITSLICED */ /* Software AES - SetKey */ static WARN_UNUSED_RESULT int wc_AesSetKeyLocal( @@ -4075,6 +4094,24 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) word32 localSz = 32; #endif + switch (keylen) { + #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 && \ + defined(WOLFSSL_AES_128) + case 16: + #endif + #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 && \ + defined(WOLFSSL_AES_192) + case 24: + #endif + #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 && \ + defined(WOLFSSL_AES_256) + case 32: + #endif + break; + default: + return BAD_FUNC_ARG; + } + #ifdef WOLFSSL_MAXQ10XX_CRYPTO if (wc_MAXQ10XX_AesSetKey(aes, userKey, keylen) != 0) { return WC_HW_E; @@ -4172,12 +4209,21 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) aes->keylen = (int)keylen; aes->rounds = (keylen/4) + 6; + ret = wc_AesSetIV(aes, iv); + if (ret != 0) + return ret; + +#ifdef WC_AES_C_DYNAMIC_FALLBACK +#ifdef NEED_AES_TABLES + AesSetKey_C(aes, userKey, keylen, dir); +#endif /* NEED_AES_TABLES */ +#endif /* WC_AES_C_DYNAMIC_FALLBACK */ #ifdef WOLFSSL_AESNI aes->use_aesni = 0; - if (checkAESNI == 0) { + if (checkedAESNI == 0) { haveAESNI = Check_CPU_support_AES(); - checkAESNI = 1; + checkedAESNI = 1; } if (haveAESNI) { #ifdef WOLFSSL_LINUXKM @@ -4185,18 +4231,33 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) if ((wc_ptr_t)&aes->key & (wc_ptr_t)0xf) { return BAD_ALIGN_E; } - #endif - aes->use_aesni = 1; - if (iv) - XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE); - else - XMEMSET(aes->reg, 0, AES_BLOCK_SIZE); - if (dir == AES_ENCRYPTION) - return AES_set_encrypt_key(userKey, (int)keylen * 8, aes); - #ifdef HAVE_AES_DECRYPT - else - return AES_set_decrypt_key(userKey, (int)keylen * 8, aes); - #endif + #endif /* WOLFSSL_LINUXKM */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + if (dir == AES_ENCRYPTION) + ret = AES_set_encrypt_key_AESNI(userKey, (int)keylen * 8, aes); +#ifdef HAVE_AES_DECRYPT + else + ret = AES_set_decrypt_key_AESNI(userKey, (int)keylen * 8, aes); +#endif + + RESTORE_VECTOR_REGISTERS(); + + if (ret == 0) + aes->use_aesni = 1; + else { +#ifdef WC_AES_C_DYNAMIC_FALLBACK + ret = 0; +#endif + } + return ret; + } else { +#ifdef WC_AES_C_DYNAMIC_FALLBACK + return 0; +#else + return ret; +#endif + } } #endif /* WOLFSSL_AESNI */ @@ -4230,10 +4291,16 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) } #endif + XMEMCPY(aes->key, userKey, keylen); + #ifndef WC_AES_BITSLICED #if defined(LITTLE_ENDIAN_ORDER) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \ (!defined(WOLFSSL_ESP32_CRYPT) || \ defined(NO_WOLFSSL_ESP32_CRYPT_AES)) + + /* software */ + ByteReverseWords(aes->key, aes->key, keylen); + #elif defined(WOLFSSL_ESP32_CRYPT) && !defined(NO_WOLFSSL_ESP32_CRYPT_AES) if (wc_esp32AesSupportedKeyLen(aes)) { /* supported lengths don't get reversed */ @@ -4263,28 +4330,11 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) return WC_HW_E; } #endif -#endif +#endif /* !WC_AES_BITSLICED */ - #ifdef NEED_AES_TABLES - switch (keylen) { - #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 && \ - defined(WOLFSSL_AES_128) - case 16: - #endif - #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 && \ - defined(WOLFSSL_AES_192) - case 24: - #endif - #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 && \ - defined(WOLFSSL_AES_256) - case 32: - #endif - break; - default: - return BAD_FUNC_ARG; - } +#ifdef NEED_AES_TABLES AesSetKey_C(aes, userKey, keylen, dir); - #endif /* NEED_AES_TABLES */ +#endif /* NEED_AES_TABLES */ #if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES) XMEMCPY((byte*)aes->key, userKey, keylen); @@ -4371,6 +4421,50 @@ int wc_AesSetIV(Aes* aes, const byte* iv) return 0; } +#ifdef WOLFSSL_AESNI + +#ifdef WC_AES_C_DYNAMIC_FALLBACK + +#define VECTOR_REGISTERS_PUSH { \ + int orig_use_aesni = aes->use_aesni; \ + if (aes->use_aesni && (SAVE_VECTOR_REGISTERS2() != 0)) { \ + aes->use_aesni = 0; \ + } \ + WC_DO_NOTHING + +#define VECTOR_REGISTERS_POP \ + if (aes->use_aesni) \ + RESTORE_VECTOR_REGISTERS(); \ + else \ + aes->use_aesni = orig_use_aesni; \ + } \ + WC_DO_NOTHING + +#else + +#define VECTOR_REGISTERS_PUSH { \ + if (aes->use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) { \ + return ret; \ + } \ + WC_DO_NOTHING + +#define VECTOR_REGISTERS_POP \ + if (aes->use_aesni) { \ + RESTORE_VECTOR_REGISTERS(); \ + } \ + } \ + WC_DO_NOTHING + +#endif + +#else /* !WOLFSSL_AESNI */ + +#define VECTOR_REGISTERS_PUSH { WC_DO_NOTHING +#define VECTOR_REGISTERS_POP } WC_DO_NOTHING + +#endif /* !WOLFSSL_AESNI */ + + /* AES-DIRECT */ #if defined(WOLFSSL_AES_DIRECT) #if defined(HAVE_COLDFIRE_SEC) @@ -4386,53 +4480,43 @@ int wc_AesSetIV(Aes* aes, const byte* iv) #elif defined(WOLFSSL_DEVCRYPTO_AES) /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */ - #elif defined(WOLFSSL_LINUXKM) && defined(WOLFSSL_AESNI) + #else - WARN_UNUSED_RESULT int wc_AesEncryptDirect( - Aes* aes, byte* out, const byte* in) + /* Allow direct access to one block encrypt */ + int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in) { int ret; - if (haveAESNI && aes->use_aesni) - SAVE_VECTOR_REGISTERS(return _svr_ret;); + + if (aes == NULL) + return BAD_FUNC_ARG; + VECTOR_REGISTERS_PUSH; ret = wc_AesEncrypt(aes, in, out); - if (haveAESNI && aes->use_aesni) - RESTORE_VECTOR_REGISTERS(); + VECTOR_REGISTERS_POP; return ret; } + /* vector reg save/restore is explicit in all below calls to * wc_Aes{En,De}cryptDirect(), so bypass the public version with a * macro. */ #define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out) + #ifdef HAVE_AES_DECRYPT /* Allow direct access to one block decrypt */ - WARN_UNUSED_RESULT int wc_AesDecryptDirect( - Aes* aes, byte* out, const byte* in) + int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in) { int ret; - if (haveAESNI && aes->use_aesni) - SAVE_VECTOR_REGISTERS(return _svr_ret;); + + if (aes == NULL) + return BAD_FUNC_ARG; + VECTOR_REGISTERS_PUSH; ret = wc_AesDecrypt(aes, in, out); - if (haveAESNI && aes->use_aesni) - RESTORE_VECTOR_REGISTERS(); + VECTOR_REGISTERS_POP; return ret; } - #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out) - #endif /* HAVE_AES_DECRYPT */ - #else + #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out) - /* Allow direct access to one block encrypt */ - int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in) - { - return wc_AesEncrypt(aes, in, out); - } - #ifdef HAVE_AES_DECRYPT - /* Allow direct access to one block decrypt */ - int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in) - { - return wc_AesDecrypt(aes, in, out); - } #endif /* HAVE_AES_DECRYPT */ #endif /* AES direct block */ #endif /* WOLFSSL_AES_DIRECT */ @@ -5135,6 +5219,7 @@ int wc_AesSetIV(Aes* aes, const byte* iv) int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) { word32 blocks; + int ret; if (aes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; @@ -5198,10 +5283,23 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) return se050_aes_crypt(aes, in, out, sz, AES_ENCRYPTION, kAlgorithm_SSS_AES_CBC); } - #endif - - #ifdef WOLFSSL_AESNI - if (haveAESNI) { + else + #elif defined(WOLFSSL_ESPIDF) && defined(NEED_AESCBC_HW_FALLBACK) + if (wc_esp32AesSupportedKeyLen(aes)) { + ESP_LOGV(TAG, "wc_AesCbcEncrypt calling wc_esp32AesCbcEncrypt"); + return wc_esp32AesCbcEncrypt(aes, out, in, sz); + } + else { + /* For example, the ESP32-S3 does not support HW for len = 24, + * so fall back to SW */ + #ifdef DEBUG_WOLFSSL + ESP_LOGW(TAG, "wc_AesCbcEncrypt HW Falling back, " + "unsupported keylen = %d", aes->keylen); + #endif + } + #elif defined(WOLFSSL_AESNI) + VECTOR_REGISTERS_PUSH; + if (aes->use_aesni) { #ifdef DEBUG_AESNI printf("about to aes cbc encrypt\n"); printf("in = %p\n", in); @@ -5218,66 +5316,55 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) byte* tmp = (byte*)XMALLOC(sz + AES_BLOCK_SIZE + AESNI_ALIGN, aes->heap, DYNAMIC_TYPE_TMP_BUFFER); byte* tmp_align; - if (tmp == NULL) return MEMORY_E; - - tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN)); - XMEMCPY(tmp_align, in, sz); - SAVE_VECTOR_REGISTERS(XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER); return _svr_ret;); - AES_CBC_encrypt(tmp_align, tmp_align, (byte*)aes->reg, sz, - (byte*)aes->key, (int)aes->rounds); - RESTORE_VECTOR_REGISTERS(); - /* store iv for next call */ - XMEMCPY(aes->reg, tmp_align + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); - - XMEMCPY(out, tmp_align, sz); - XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER); - return 0; + if (tmp == NULL) + ret = MEMORY_E; + else { + tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN)); + XMEMCPY(tmp_align, in, sz); + AES_CBC_encrypt_AESNI(tmp_align, tmp_align, (byte*)aes->reg, sz, + (byte*)aes->key, (int)aes->rounds); + /* store iv for next call */ + XMEMCPY(aes->reg, tmp_align + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); + + XMEMCPY(out, tmp_align, sz); + XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER); + ret = 0; + } #else WOLFSSL_MSG("AES-CBC encrypt with bad alignment"); WOLFSSL_ERROR_VERBOSE(BAD_ALIGN_E); - return BAD_ALIGN_E; + ret = BAD_ALIGN_E; #endif - } - - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_CBC_encrypt(in, out, (byte*)aes->reg, sz, (byte*)aes->key, - (int)aes->rounds); - RESTORE_VECTOR_REGISTERS(); - /* store iv for next call */ - XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); + } else { + AES_CBC_encrypt_AESNI(in, out, (byte*)aes->reg, sz, (byte*)aes->key, + (int)aes->rounds); + /* store iv for next call */ + XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); - return 0; + ret = 0; + } } + else #endif + { + ret = 0; + while (blocks--) { + xorbuf((byte*)aes->reg, in, AES_BLOCK_SIZE); + ret = wc_AesEncrypt(aes, (byte*)aes->reg, (byte*)aes->reg); + if (ret != 0) + break; + XMEMCPY(out, aes->reg, AES_BLOCK_SIZE); - #if defined(WOLFSSL_ESPIDF) && defined(NEED_AESCBC_HW_FALLBACK) - if (wc_esp32AesSupportedKeyLen(aes)) { - ESP_LOGV(TAG, "wc_AesCbcEncrypt calling wc_esp32AesCbcEncrypt"); - return wc_esp32AesCbcEncrypt(aes, out, in, sz); - } - else { - /* For example, the ESP32-S3 does not support HW for len = 24, - * so fall back to SW */ - #ifdef DEBUG_WOLFSSL - ESP_LOGW(TAG, "wc_AesCbcEncrypt HW Falling back, " - "unsupported keylen = %d", aes->keylen); - #endif + out += AES_BLOCK_SIZE; + in += AES_BLOCK_SIZE; + } } - #endif - - while (blocks--) { - int ret; - xorbuf((byte*)aes->reg, in, AES_BLOCK_SIZE); - ret = wc_AesEncrypt(aes, (byte*)aes->reg, (byte*)aes->reg); - if (ret != 0) - return ret; - XMEMCPY(out, aes->reg, AES_BLOCK_SIZE); - out += AES_BLOCK_SIZE; - in += AES_BLOCK_SIZE; - } + #ifdef WOLFSSL_AESNI + VECTOR_REGISTERS_POP; + #endif - return 0; + return ret; } /* wc_AesCbcEncrypt */ #ifdef HAVE_AES_DECRYPT @@ -5285,6 +5372,8 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz) { word32 blocks; + int ret; + if (aes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } @@ -5365,8 +5454,10 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) } #endif + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - if (haveAESNI) { + if (aes->use_aesni) { #ifdef DEBUG_AESNI printf("about to aes cbc decrypt\n"); printf("in = %p\n", in); @@ -5379,105 +5470,103 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) /* if input and output same will overwrite input iv */ XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE); - SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(WOLFSSL_AESNI_BY4) || defined(WOLFSSL_X86_BUILD) - AES_CBC_decrypt_by4(in, out, (byte*)aes->reg, sz, (byte*)aes->key, + AES_CBC_decrypt_AESNI_by4(in, out, (byte*)aes->reg, sz, (byte*)aes->key, aes->rounds); #elif defined(WOLFSSL_AESNI_BY6) - AES_CBC_decrypt_by6(in, out, (byte*)aes->reg, sz, (byte*)aes->key, + AES_CBC_decrypt_AESNI_by6(in, out, (byte*)aes->reg, sz, (byte*)aes->key, aes->rounds); #else /* WOLFSSL_AESNI_BYx */ - AES_CBC_decrypt_by8(in, out, (byte*)aes->reg, sz, (byte*)aes->key, + AES_CBC_decrypt_AESNI_by8(in, out, (byte*)aes->reg, sz, (byte*)aes->key, (int)aes->rounds); #endif /* WOLFSSL_AESNI_BYx */ /* store iv for next call */ - RESTORE_VECTOR_REGISTERS(); XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); - return 0; + ret = 0; } + else #endif - + { + ret = 0; #ifdef WC_AES_BITSLICED - if (in != out) { - unsigned char dec[AES_BLOCK_SIZE * BS_WORD_SIZE]; - - while (blocks > BS_WORD_SIZE) { - AesDecryptBlocks_C(aes, in, dec, AES_BLOCK_SIZE * BS_WORD_SIZE); - xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); - xorbufout(out + AES_BLOCK_SIZE, dec + AES_BLOCK_SIZE, in, - AES_BLOCK_SIZE * (BS_WORD_SIZE - 1)); - XMEMCPY(aes->reg, in + (AES_BLOCK_SIZE * (BS_WORD_SIZE - 1)), - AES_BLOCK_SIZE); - in += AES_BLOCK_SIZE * BS_WORD_SIZE; - out += AES_BLOCK_SIZE * BS_WORD_SIZE; - blocks -= BS_WORD_SIZE; - } - if (blocks > 0) { - AesDecryptBlocks_C(aes, in, dec, blocks * AES_BLOCK_SIZE); - xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); - xorbufout(out + AES_BLOCK_SIZE, dec + AES_BLOCK_SIZE, in, - AES_BLOCK_SIZE * (blocks - 1)); - XMEMCPY(aes->reg, in + (AES_BLOCK_SIZE * (blocks - 1)), - AES_BLOCK_SIZE); - blocks = 0; + if (in != out) { + unsigned char dec[AES_BLOCK_SIZE * BS_WORD_SIZE]; + + while (blocks > BS_WORD_SIZE) { + AesDecryptBlocks_C(aes, in, dec, AES_BLOCK_SIZE * BS_WORD_SIZE); + xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); + xorbufout(out + AES_BLOCK_SIZE, dec + AES_BLOCK_SIZE, in, + AES_BLOCK_SIZE * (BS_WORD_SIZE - 1)); + XMEMCPY(aes->reg, in + (AES_BLOCK_SIZE * (BS_WORD_SIZE - 1)), + AES_BLOCK_SIZE); + in += AES_BLOCK_SIZE * BS_WORD_SIZE; + out += AES_BLOCK_SIZE * BS_WORD_SIZE; + blocks -= BS_WORD_SIZE; + } + if (blocks > 0) { + AesDecryptBlocks_C(aes, in, dec, blocks * AES_BLOCK_SIZE); + xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); + xorbufout(out + AES_BLOCK_SIZE, dec + AES_BLOCK_SIZE, in, + AES_BLOCK_SIZE * (blocks - 1)); + XMEMCPY(aes->reg, in + (AES_BLOCK_SIZE * (blocks - 1)), + AES_BLOCK_SIZE); + blocks = 0; + } } - } - else { - unsigned char dec[AES_BLOCK_SIZE * BS_WORD_SIZE]; - int i; + else { + unsigned char dec[AES_BLOCK_SIZE * BS_WORD_SIZE]; + int i; + + while (blocks > BS_WORD_SIZE) { + AesDecryptBlocks_C(aes, in, dec, AES_BLOCK_SIZE * BS_WORD_SIZE); + XMEMCPY(aes->tmp, in + (BS_WORD_SIZE - 1) * AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + for (i = BS_WORD_SIZE-1; i >= 1; i--) { + xorbufout(out + i * AES_BLOCK_SIZE, + dec + i * AES_BLOCK_SIZE, in + (i - 1) * AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } + xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); + XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); - while (blocks > BS_WORD_SIZE) { - AesDecryptBlocks_C(aes, in, dec, AES_BLOCK_SIZE * BS_WORD_SIZE); - XMEMCPY(aes->tmp, in + (BS_WORD_SIZE - 1) * AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - for (i = BS_WORD_SIZE-1; i >= 1; i--) { - xorbufout(out + i * AES_BLOCK_SIZE, - dec + i * AES_BLOCK_SIZE, in + (i - 1) * AES_BLOCK_SIZE, - AES_BLOCK_SIZE); + in += AES_BLOCK_SIZE * BS_WORD_SIZE; + out += AES_BLOCK_SIZE * BS_WORD_SIZE; + blocks -= BS_WORD_SIZE; } - xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); - XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); + if (blocks > 0) { + AesDecryptBlocks_C(aes, in, dec, blocks * AES_BLOCK_SIZE); + XMEMCPY(aes->tmp, in + (blocks - 1) * AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + for (i = blocks-1; i >= 1; i--) { + xorbufout(out + i * AES_BLOCK_SIZE, + dec + i * AES_BLOCK_SIZE, in + (i - 1) * AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } + xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); + XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); - in += AES_BLOCK_SIZE * BS_WORD_SIZE; - out += AES_BLOCK_SIZE * BS_WORD_SIZE; - blocks -= BS_WORD_SIZE; - } - if (blocks > 0) { - AesDecryptBlocks_C(aes, in, dec, blocks * AES_BLOCK_SIZE); - XMEMCPY(aes->tmp, in + (blocks - 1) * AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - for (i = blocks-1; i >= 1; i--) { - xorbufout(out + i * AES_BLOCK_SIZE, - dec + i * AES_BLOCK_SIZE, in + (i - 1) * AES_BLOCK_SIZE, - AES_BLOCK_SIZE); + blocks = 0; } - xorbufout(out, dec, aes->reg, AES_BLOCK_SIZE); + } +#else + while (blocks--) { + XMEMCPY(aes->tmp, in, AES_BLOCK_SIZE); + ret = wc_AesDecrypt(aes, in, out); + if (ret != 0) + return ret; + xorbuf(out, (byte*)aes->reg, AES_BLOCK_SIZE); + /* store iv for next call */ XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); - blocks = 0; + out += AES_BLOCK_SIZE; + in += AES_BLOCK_SIZE; } - } -#else - while (blocks--) { - int ret; -#ifdef WOLFSSL_AESNI - ret = wc_AesDecrypt(aes, in, out); -#else - XMEMCPY(aes->tmp, in, AES_BLOCK_SIZE); - ret = wc_AesDecrypt(aes, (byte*)aes->tmp, out); #endif - if (ret != 0) - return ret; - xorbuf(out, (byte*)aes->reg, AES_BLOCK_SIZE); - /* store iv for next call */ - XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE); - - out += AES_BLOCK_SIZE; - in += AES_BLOCK_SIZE; } -#endif - return 0; + VECTOR_REGISTERS_POP; + + return ret; } #endif /* HAVE_AES_DECRYPT */ @@ -5704,7 +5793,7 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) { byte scratch[AES_BLOCK_SIZE]; - int ret; + int ret = 0; word32 processed; if (aes == NULL || out == NULL || in == NULL) { @@ -5732,6 +5821,8 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) aes->left -= processed; sz -= processed; + VECTOR_REGISTERS_PUSH; + #if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \ !defined(XTRANSFORM_AESCTRBLOCK) if (in != out && sz >= AES_BLOCK_SIZE) { @@ -5765,13 +5856,8 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) XTRANSFORM_AESCTRBLOCK(aes, out, in); #else ret = wc_AesEncrypt(aes, (byte*)aes->reg, scratch); - if (ret != 0) { - ForceZero(scratch, AES_BLOCK_SIZE); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(scratch, AES_BLOCK_SIZE); - #endif - return ret; - } + if (ret != 0) + break; xorbuf(scratch, in, AES_BLOCK_SIZE); XMEMCPY(out, scratch, AES_BLOCK_SIZE); #endif @@ -5786,25 +5872,25 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) } /* handle non block size remaining and store unused byte count in left */ - if (sz) { + if ((ret == 0) && sz) { ret = wc_AesEncrypt(aes, (byte*)aes->reg, (byte*)aes->tmp); - if (ret != 0) { - ForceZero(scratch, AES_BLOCK_SIZE); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(scratch, AES_BLOCK_SIZE); - #endif - return ret; + if (ret == 0) { + IncrementAesCounter((byte*)aes->reg); + aes->left = AES_BLOCK_SIZE - sz; + xorbufout(out, in, aes->tmp, sz); } - IncrementAesCounter((byte*)aes->reg); - - aes->left = AES_BLOCK_SIZE - sz; - xorbufout(out, in, aes->tmp, sz); } + if (ret < 0) + ForceZero(scratch, AES_BLOCK_SIZE); + #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(scratch, AES_BLOCK_SIZE); #endif - return 0; + + VECTOR_REGISTERS_POP; + + return ret; } int wc_AesCtrSetKey(Aes* aes, const byte* key, word32 len, @@ -6070,11 +6156,6 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #ifdef WOLFSSL_AESGCM_STREAM aes->gcmKeySet = 1; #endif - #ifdef WOLFSSL_AESNI - /* AES-NI code generates its own H value. */ - if (haveAESNI) - return ret; - #endif /* WOLFSSL_AESNI */ #if defined(WOLFSSL_SECO_CAAM) if (aes->devId == WOLFSSL_SECO_DEVID) { return ret; @@ -6087,8 +6168,14 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #endif /* WOLFSSL_RENESAS_RSIP && WOLFSSL_RENESAS_FSPSM_CRYPTONLY*/ #if !defined(FREESCALE_LTC_AES_GCM) - if (ret == 0) + if (ret == 0) { + VECTOR_REGISTERS_PUSH; + /* AES-NI code generates its own H value, but generate it here too, to + * assure pure-C fallback is always usable. + */ ret = wc_AesEncrypt(aes, iv, aes->gcm.H); + VECTOR_REGISTERS_POP; + } if (ret == 0) { #if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) GenerateM0(&aes->gcm); @@ -6120,12 +6207,12 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #define HAVE_INTEL_AVX2 #endif /* USE_INTEL_SPEEDUP */ -void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, +void AES_GCM_encrypt_aesni(const unsigned char *in, unsigned char *out, const unsigned char* addt, const unsigned char* ivec, unsigned char *tag, word32 nbytes, word32 abytes, word32 ibytes, word32 tbytes, const unsigned char* key, int nr) - XASM_LINK("AES_GCM_encrypt"); + XASM_LINK("AES_GCM_encrypt_aesni"); #ifdef HAVE_INTEL_AVX1 void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out, const unsigned char* addt, const unsigned char* ivec, @@ -6146,12 +6233,12 @@ void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, #endif /* HAVE_INTEL_AVX1 */ #ifdef HAVE_AES_DECRYPT -void AES_GCM_decrypt(const unsigned char *in, unsigned char *out, +void AES_GCM_decrypt_aesni(const unsigned char *in, unsigned char *out, const unsigned char* addt, const unsigned char* ivec, const unsigned char *tag, word32 nbytes, word32 abytes, word32 ibytes, word32 tbytes, const unsigned char* key, int nr, int* res) - XASM_LINK("AES_GCM_decrypt"); + XASM_LINK("AES_GCM_decrypt_aesni"); #ifdef HAVE_INTEL_AVX1 void AES_GCM_decrypt_avx1(const unsigned char *in, unsigned char *out, const unsigned char* addt, const unsigned char* ivec, @@ -7948,6 +8035,8 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, byte* authTag, word32 authTagSz, const byte* authIn, word32 authInSz) { + int ret; + /* argument checks */ if (aes == NULL || authTagSz > AES_BLOCK_SIZE || ivSz == 0) { return BAD_FUNC_ARG; @@ -8022,38 +8111,41 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, authTag, authTagSz, authIn, authInSz); #endif /* STM32_CRYPTO_AES_GCM */ + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - #ifdef HAVE_INTEL_AVX2 - if (IS_INTEL_AVX2(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz, - authTagSz, (const byte*)aes->key, (int)aes->rounds); - RESTORE_VECTOR_REGISTERS(); - return 0; - } - else - #endif - #if defined(HAVE_INTEL_AVX1) - if (IS_INTEL_AVX1(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz, - authTagSz, (const byte*)aes->key, (int)aes->rounds); - RESTORE_VECTOR_REGISTERS(); - return 0; - } - else - #endif - if (haveAESNI) { - AES_GCM_encrypt(in, out, authIn, iv, authTag, sz, authInSz, ivSz, + if (aes->use_aesni) { +#ifdef HAVE_INTEL_AVX2 + if (IS_INTEL_AVX2(intel_flags)) { + AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz, + authTagSz, (const byte*)aes->key, (int)aes->rounds); + ret = 0; + } + else +#endif +#if defined(HAVE_INTEL_AVX1) + if (IS_INTEL_AVX1(intel_flags)) { + AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz, + authTagSz, (const byte*)aes->key, (int)aes->rounds); + ret = 0; + } else +#endif + { + AES_GCM_encrypt_aesni(in, out, authIn, iv, authTag, sz, authInSz, ivSz, authTagSz, (const byte*)aes->key, (int)aes->rounds); - return 0; + ret = 0; + } } else -#endif +#endif /* WOLFSSL_AESNI */ { - return AES_GCM_encrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz, - authIn, authInSz); + ret = AES_GCM_encrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz, + authIn, authInSz); } + + VECTOR_REGISTERS_POP; + + return ret; } #endif @@ -8504,6 +8596,7 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, const byte* authTag, word32 authTagSz, const byte* authIn, word32 authInSz) { + int ret; #ifdef WOLFSSL_AESNI int res = AES_GCM_AUTH_E; #endif @@ -8582,49 +8675,61 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, authTag, authTagSz, authIn, authInSz); #endif /* STM32_CRYPTO_AES_GCM */ + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - #ifdef HAVE_INTEL_AVX2 - if (IS_INTEL_AVX2(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz, - authTagSz, (byte*)aes->key, (int)aes->rounds, &res); - RESTORE_VECTOR_REGISTERS(); - if (res == 0) - return AES_GCM_AUTH_E; - return 0; - } - else - #endif - #if defined(HAVE_INTEL_AVX1) - if (IS_INTEL_AVX1(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz, - authTagSz, (byte*)aes->key, (int)aes->rounds, &res); - RESTORE_VECTOR_REGISTERS(); - if (res == 0) - return AES_GCM_AUTH_E; - return 0; - } - else - #endif - if (haveAESNI) { - AES_GCM_decrypt(in, out, authIn, iv, authTag, sz, authInSz, ivSz, + if (aes->use_aesni) { +#ifdef HAVE_INTEL_AVX2 + if (IS_INTEL_AVX2(intel_flags)) { + AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz, + authTagSz, (byte*)aes->key, (int)aes->rounds, &res); + if (res == 0) + ret = AES_GCM_AUTH_E; + else + ret = 0; + } + else +#endif +#if defined(HAVE_INTEL_AVX1) + if (IS_INTEL_AVX1(intel_flags)) { + AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz, + authTagSz, (byte*)aes->key, (int)aes->rounds, &res); + if (res == 0) + ret = AES_GCM_AUTH_E; + else + ret = 0; + } + else +#endif + { + AES_GCM_decrypt_aesni(in, out, authIn, iv, authTag, sz, authInSz, ivSz, authTagSz, (byte*)aes->key, (int)aes->rounds, &res); - if (res == 0) - return AES_GCM_AUTH_E; - return 0; + if (res == 0) + ret = AES_GCM_AUTH_E; + else + ret = 0; + } } else -#endif +#endif /* WOLFSSL_AESNI */ { - return AES_GCM_decrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz, - authIn, authInSz); + ret = AES_GCM_decrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz, + authIn, authInSz); } + + VECTOR_REGISTERS_POP; + + return ret; } #endif #endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */ #ifdef WOLFSSL_AESGCM_STREAM + +#if defined(WC_AES_C_DYNAMIC_FALLBACK) && defined(WOLFSSL_AESNI) + #error "AES-GCM streaming with AESNI is incompatible with WC_AES_C_DYNAMIC_FALLBACK." +#endif + /* Initialize the AES GCM cipher with an IV. C implementation. * * @param [in, out] aes AES object. @@ -8636,6 +8741,10 @@ static WARN_UNUSED_RESULT int AesGcmInit_C(Aes* aes, const byte* iv, word32 ivSz ALIGN32 byte counter[AES_BLOCK_SIZE]; int ret; +#ifdef WOLFSSL_AESNI + aes->use_aesni = 0; +#endif + if (ivSz == GCM_NONCE_MID_SZ) { /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */ XMEMCPY(counter, iv, ivSz); @@ -8859,6 +8968,8 @@ extern void AES_GCM_encrypt_final_aesni(unsigned char* tag, static WARN_UNUSED_RESULT int AesGcmInit_aesni( Aes* aes, const byte* iv, word32 ivSz) { + ASSERT_SAVED_VECTOR_REGISTERS(); + /* Reset state fields. */ aes->aSz = 0; aes->cSz = 0; @@ -8870,28 +8981,25 @@ static WARN_UNUSED_RESULT int AesGcmInit_aesni( #ifdef HAVE_INTEL_AVX2 if (IS_INTEL_AVX2(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_GCM_init_avx2((byte*)aes->key, (int)aes->rounds, iv, ivSz, aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes)); - RESTORE_VECTOR_REGISTERS(); } else #endif #ifdef HAVE_INTEL_AVX1 if (IS_INTEL_AVX1(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_GCM_init_avx1((byte*)aes->key, (int)aes->rounds, iv, ivSz, aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes)); - RESTORE_VECTOR_REGISTERS(); } else #endif { - SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_GCM_init_aesni((byte*)aes->key, (int)aes->rounds, iv, ivSz, aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes)); - RESTORE_VECTOR_REGISTERS(); } + + aes->use_aesni = 1; + return 0; } @@ -9034,7 +9142,8 @@ static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_aesni( int partial; int ret; - SAVE_VECTOR_REGISTERS(return _svr_ret;); + ASSERT_SAVED_VECTOR_REGISTERS(); + /* Hash in A, the Authentication Data */ ret = AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL)); if (ret != 0) @@ -9144,7 +9253,6 @@ static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_aesni( aes->cOver = (byte)partial; } } - RESTORE_VECTOR_REGISTERS(); return 0; } @@ -9163,7 +9271,8 @@ static WARN_UNUSED_RESULT int AesGcmEncryptFinal_aesni( /* AAD block incomplete when > 0 */ byte over = aes->aOver; - SAVE_VECTOR_REGISTERS(return _svr_ret;); + ASSERT_SAVED_VECTOR_REGISTERS(); + if (aes->cOver > 0) { /* Cipher text block incomplete. */ over = aes->cOver; @@ -9210,7 +9319,7 @@ static WARN_UNUSED_RESULT int AesGcmEncryptFinal_aesni( AES_GCM_encrypt_final_aesni(AES_TAG(aes), authTag, authTagSz, aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes)); } - RESTORE_VECTOR_REGISTERS(); + return 0; } @@ -9264,7 +9373,8 @@ static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_aesni( int partial; int ret; - SAVE_VECTOR_REGISTERS(return _svr_ret;); + ASSERT_SAVED_VECTOR_REGISTERS(); + /* Hash in A, the Authentication Data */ ret = AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL)); if (ret != 0) @@ -9376,7 +9486,7 @@ static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_aesni( aes->cOver = (byte)partial; } } - RESTORE_VECTOR_REGISTERS(); + return 0; } @@ -9400,7 +9510,8 @@ static WARN_UNUSED_RESULT int AesGcmDecryptFinal_aesni( byte over = aes->aOver; byte *lastBlock = AES_LASTGBLOCK(aes); - SAVE_VECTOR_REGISTERS(return _svr_ret;); + ASSERT_SAVED_VECTOR_REGISTERS(); + if (aes->cOver > 0) { /* Cipher text block incomplete. */ over = aes->cOver; @@ -9445,7 +9556,7 @@ static WARN_UNUSED_RESULT int AesGcmDecryptFinal_aesni( AES_GCM_decrypt_final_aesni(AES_TAG(aes), authTag, authTagSz, aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res); } - RESTORE_VECTOR_REGISTERS(); + /* Return error code when calculated doesn't match input. */ if (res == 0) { ret = AES_GCM_AUTH_E; @@ -9514,15 +9625,10 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv, if (iv != NULL) { /* Initialize with the IV. */ + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - if (haveAESNI - #ifdef HAVE_INTEL_AVX2 - || IS_INTEL_AVX2(intel_flags) - #endif - #ifdef HAVE_INTEL_AVX1 - || IS_INTEL_AVX1(intel_flags) - #endif - ) { + if (aes->use_aesni) { ret = AesGcmInit_aesni(aes, iv, ivSz); } else @@ -9531,7 +9637,10 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv, ret = AesGcmInit_C(aes, iv, ivSz); } - aes->nonceSet = 1; + VECTOR_REGISTERS_POP; + + if (ret == 0) + aes->nonceSet = 1; } } @@ -9643,15 +9752,10 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, if (ret == 0) { /* Encrypt with AAD and/or plaintext. */ - #if defined(WOLFSSL_AESNI) - if (haveAESNI - #ifdef HAVE_INTEL_AVX2 - || IS_INTEL_AVX2(intel_flags) - #endif - #ifdef HAVE_INTEL_AVX1 - || IS_INTEL_AVX1(intel_flags) - #endif - ) { + VECTOR_REGISTERS_PUSH; + + #ifdef WOLFSSL_AESNI + if (aes->use_aesni) { ret = AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz); } else @@ -9659,12 +9763,14 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, { /* Encrypt the plaintext. */ ret = AesGcmCryptUpdate_C(aes, out, in, sz); - if (ret != 0) - return ret; - /* Update the authentication tag with any authentication data and the - * new cipher text. */ - GHASH_UPDATE(aes, authIn, authInSz, out, sz); + if (ret == 0) { + /* Update the authentication tag with any authentication data and the + * new cipher text. */ + GHASH_UPDATE(aes, authIn, authInSz, out, sz); + } } + + VECTOR_REGISTERS_POP; } return ret; @@ -9701,15 +9807,9 @@ int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz) if (ret == 0) { /* Calculate authentication tag. */ + VECTOR_REGISTERS_PUSH; #ifdef WOLFSSL_AESNI - if (haveAESNI - #ifdef HAVE_INTEL_AVX2 - || IS_INTEL_AVX2(intel_flags) - #endif - #ifdef HAVE_INTEL_AVX1 - || IS_INTEL_AVX1(intel_flags) - #endif - ) { + if (aes->use_aesni) { ret = AesGcmEncryptFinal_aesni(aes, authTag, authTagSz); } else @@ -9717,6 +9817,7 @@ int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz) { ret = AesGcmFinal_C(aes, authTag, authTagSz); } + VECTOR_REGISTERS_POP; } if ((ret == 0) && aes->ctrSet) { @@ -9789,15 +9890,9 @@ int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, if (ret == 0) { /* Decrypt with AAD and/or cipher text. */ - #if defined(WOLFSSL_AESNI) - if (haveAESNI - #ifdef HAVE_INTEL_AVX2 - || IS_INTEL_AVX2(intel_flags) - #endif - #ifdef HAVE_INTEL_AVX1 - || IS_INTEL_AVX1(intel_flags) - #endif - ) { + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI + if (aes->use_aesni) { ret = AesGcmDecryptUpdate_aesni(aes, out, in, sz, authIn, authInSz); } else @@ -9809,6 +9904,7 @@ int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, /* Decrypt the cipher text. */ ret = AesGcmCryptUpdate_C(aes, out, in, sz); } + VECTOR_REGISTERS_POP; } return ret; @@ -9845,15 +9941,9 @@ int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz) if (ret == 0) { /* Calculate authentication tag and compare with one passed in.. */ + VECTOR_REGISTERS_PUSH; #ifdef WOLFSSL_AESNI - if (haveAESNI - #ifdef HAVE_INTEL_AVX2 - || IS_INTEL_AVX2(intel_flags) - #endif - #ifdef HAVE_INTEL_AVX1 - || IS_INTEL_AVX1(intel_flags) - #endif - ) { + if (aes->use_aesni) { ret = AesGcmDecryptFinal_aesni(aes, authTag, authTagSz); } else @@ -9869,6 +9959,7 @@ int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz) } } } + VECTOR_REGISTERS_POP; } /* reset the state */ @@ -10393,12 +10484,12 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, byte* authTag, word32 authTagSz, const byte* authIn, word32 authInSz) { -#ifndef WOLFSSL_AESNI - byte A[AES_BLOCK_SIZE]; - byte B[AES_BLOCK_SIZE]; -#else +#ifdef WOLFSSL_AESNI ALIGN128 byte A[AES_BLOCK_SIZE * 4]; ALIGN128 byte B[AES_BLOCK_SIZE * 4]; +#else + byte A[AES_BLOCK_SIZE]; + byte B[AES_BLOCK_SIZE]; #endif byte lenSz; word32 i; @@ -10447,67 +10538,38 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(B)); #endif + VECTOR_REGISTERS_PUSH; ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(A)); + if (ret == 0) + wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(A)); #endif - if (authInSz > 0) { + if ((ret == 0) && (authInSz > 0)) ret = roll_auth(aes, authIn, authInSz, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } - } - if (inSz > 0) { + + if ((ret == 0) && (inSz > 0)) ret = roll_x(aes, in, inSz, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } - } - XMEMCPY(authTag, A, authTagSz); - B[0] = lenSz - 1; - for (i = 0; i < lenSz; i++) - B[AES_BLOCK_SIZE - 1 - i] = 0; - ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; + if (ret == 0) { + XMEMCPY(authTag, A, authTagSz); + + B[0] = lenSz - 1; + for (i = 0; i < lenSz; i++) + B[AES_BLOCK_SIZE - 1 - i] = 0; + ret = wc_AesEncrypt(aes, B, A); } - xorbuf(authTag, A, authTagSz); - B[15] = 1; + if (ret == 0) { + xorbuf(authTag, A, authTagSz); + B[15] = 1; + } #ifdef WOLFSSL_AESNI - if (haveAESNI && aes->use_aesni) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); + if ((ret == 0) && aes->use_aesni) { while (inSz >= AES_BLOCK_SIZE * 4) { AesCcmCtrIncSet4(B, lenSz); - AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key, + AES_ECB_encrypt_AESNI(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key, (int)aes->rounds); xorbuf(A, in, AES_BLOCK_SIZE * 4); @@ -10519,39 +10581,26 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, AesCcmCtrInc4(B, lenSz); } - RESTORE_VECTOR_REGISTERS(); } #endif - while (inSz >= AES_BLOCK_SIZE) { - ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } - xorbuf(A, in, AES_BLOCK_SIZE); - XMEMCPY(out, A, AES_BLOCK_SIZE); + if (ret == 0) { + while (inSz >= AES_BLOCK_SIZE) { + ret = wc_AesEncrypt(aes, B, A); + if (ret != 0) + break; + xorbuf(A, in, AES_BLOCK_SIZE); + XMEMCPY(out, A, AES_BLOCK_SIZE); - AesCcmCtrInc(B, lenSz); - inSz -= AES_BLOCK_SIZE; - in += AES_BLOCK_SIZE; - out += AES_BLOCK_SIZE; + AesCcmCtrInc(B, lenSz); + inSz -= AES_BLOCK_SIZE; + in += AES_BLOCK_SIZE; + out += AES_BLOCK_SIZE; + } } - if (inSz > 0) { + if ((ret == 0) && (inSz > 0)) { ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } + } + if ((ret == 0) && (inSz > 0)) { xorbuf(A, in, inSz); XMEMCPY(out, A, inSz); } @@ -10564,7 +10613,9 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, wc_MemZero_Check(B, sizeof(B)); #endif - return 0; + VECTOR_REGISTERS_POP; + + return ret; } #ifdef HAVE_AES_DECRYPT @@ -10574,19 +10625,19 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, const byte* authTag, word32 authTagSz, const byte* authIn, word32 authInSz) { -#ifndef WOLFSSL_AESNI - byte A[AES_BLOCK_SIZE]; - byte B[AES_BLOCK_SIZE]; -#else +#ifdef WOLFSSL_AESNI ALIGN128 byte B[AES_BLOCK_SIZE * 4]; ALIGN128 byte A[AES_BLOCK_SIZE * 4]; +#else + byte A[AES_BLOCK_SIZE]; + byte B[AES_BLOCK_SIZE]; #endif byte* o; byte lenSz; word32 i, oSz; byte mask = 0xFF; const word32 wordSz = (word32)sizeof(word32); - int ret; + int ret = 0; /* sanity check on arguments */ if (aes == NULL || (inSz != 0 && (in == NULL || out == NULL)) || @@ -10629,13 +10680,14 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(B)); #endif + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - if (haveAESNI && aes->use_aesni) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); + if (aes->use_aesni) { while (oSz >= AES_BLOCK_SIZE * 4) { AesCcmCtrIncSet4(B, lenSz); - AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key, + AES_ECB_encrypt_AESNI(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key, (int)aes->rounds); xorbuf(A, in, AES_BLOCK_SIZE * 4); @@ -10647,131 +10699,79 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, AesCcmCtrInc4(B, lenSz); } - RESTORE_VECTOR_REGISTERS(); } #endif + while (oSz >= AES_BLOCK_SIZE) { ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } + if (ret != 0) + break; xorbuf(A, in, AES_BLOCK_SIZE); XMEMCPY(o, A, AES_BLOCK_SIZE); - AesCcmCtrInc(B, lenSz); oSz -= AES_BLOCK_SIZE; in += AES_BLOCK_SIZE; o += AES_BLOCK_SIZE; } - if (inSz > 0) { + + if ((ret == 0) && (inSz > 0)) ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } + + if ((ret == 0) && (inSz > 0)) { xorbuf(A, in, oSz); XMEMCPY(o, A, oSz); + for (i = 0; i < lenSz; i++) + B[AES_BLOCK_SIZE - 1 - i] = 0; + ret = wc_AesEncrypt(aes, B, A); } - for (i = 0; i < lenSz; i++) - B[AES_BLOCK_SIZE - 1 - i] = 0; - ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } - - o = out; - oSz = inSz; + if (ret == 0) { + o = out; + oSz = inSz; - B[0] = (byte)((authInSz > 0 ? 64 : 0) - + (8 * (((byte)authTagSz - 2) / 2)) - + (lenSz - 1)); - for (i = 0; i < lenSz; i++) { - if (mask && i >= wordSz) - mask = 0x00; - B[AES_BLOCK_SIZE - 1 - i] = (byte)((inSz >> ((8 * i) & mask)) & mask); - } + B[0] = (byte)((authInSz > 0 ? 64 : 0) + + (8 * (((byte)authTagSz - 2) / 2)) + + (lenSz - 1)); + for (i = 0; i < lenSz; i++) { + if (mask && i >= wordSz) + mask = 0x00; + B[AES_BLOCK_SIZE - 1 - i] = (byte)((inSz >> ((8 * i) & mask)) & mask); + } - ret = wc_AesEncrypt(aes, B, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; + ret = wc_AesEncrypt(aes, B, A); } - if (authInSz > 0) { - ret = roll_auth(aes, authIn, authInSz, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } + if (ret == 0) { + if (authInSz > 0) + ret = roll_auth(aes, authIn, authInSz, A); } - if (inSz > 0) { + if ((ret == 0) && (inSz > 0)) ret = roll_x(aes, o, oSz, A); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; - } - } - B[0] = lenSz - 1; - for (i = 0; i < lenSz; i++) - B[AES_BLOCK_SIZE - 1 - i] = 0; - ret = wc_AesEncrypt(aes, B, B); - if (ret != 0) { - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); - #endif - return ret; + if (ret == 0) { + B[0] = lenSz - 1; + for (i = 0; i < lenSz; i++) + B[AES_BLOCK_SIZE - 1 - i] = 0; + ret = wc_AesEncrypt(aes, B, B); } - xorbuf(A, B, authTagSz); - if (ConstantCompare(A, authTag, (int)authTagSz) != 0) { - /* If the authTag check fails, don't keep the decrypted data. - * Unfortunately, you need the decrypted data to calculate the - * check value. */ - #if defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2) && \ - defined(ACVP_VECTOR_TESTING) + if (ret == 0) + xorbuf(A, B, authTagSz); + + if (ret == 0) { + if (ConstantCompare(A, authTag, (int)authTagSz) != 0) { + /* If the authTag check fails, don't keep the decrypted data. + * Unfortunately, you need the decrypted data to calculate the + * check value. */ + #if defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2) && \ + defined(ACVP_VECTOR_TESTING) WOLFSSL_MSG("Preserve output for vector responses"); - #else + #else if (inSz > 0) XMEMSET(out, 0, inSz); - #endif - ret = AES_CCM_AUTH_E; + #endif + ret = AES_CCM_AUTH_E; + } } ForceZero(A, sizeof(A)); @@ -10783,6 +10783,8 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, wc_MemZero_Check(B, sizeof(B)); #endif + VECTOR_REGISTERS_POP; + return ret; } @@ -11128,14 +11130,17 @@ int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz) static WARN_UNUSED_RESULT int _AesEcbEncrypt( Aes* aes, byte* out, const byte* in, word32 sz) { + int ret = 0; + #ifdef WOLF_CRYPTO_CB #ifndef WOLF_CRYPTO_CB_FIND if (aes->devId != INVALID_DEVID) #endif { - int ret = wc_CryptoCb_AesEcbEncrypt(aes, out, in, sz); + ret = wc_CryptoCb_AesEcbEncrypt(aes, out, in, sz); if (ret != CRYPTOCB_UNAVAILABLE) return ret; + ret = 0; /* fall-through when unavailable */ } #endif @@ -11143,11 +11148,12 @@ static WARN_UNUSED_RESULT int _AesEcbEncrypt( if (aes->keylen == 16) return DCPAesEcbEncrypt(aes, out, in, sz); #endif + + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - if (haveAESNI && aes->use_aesni) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_ECB_encrypt(in, out, sz, (byte*)aes->key, (int)aes->rounds); - RESTORE_VECTOR_REGISTERS(); + if (aes->use_aesni) { + AES_ECB_encrypt_AESNI(in, out, sz, (byte*)aes->key, (int)aes->rounds); } else #endif @@ -11158,29 +11164,34 @@ static WARN_UNUSED_RESULT int _AesEcbEncrypt( word32 i; for (i = 0; i < sz; i += AES_BLOCK_SIZE) { - int ret = wc_AesEncryptDirect(aes, out, in); + ret = wc_AesEncryptDirect(aes, out, in); if (ret != 0) - return ret; + break; in += AES_BLOCK_SIZE; out += AES_BLOCK_SIZE; } #endif } - return 0; + VECTOR_REGISTERS_POP; + + return ret; } static WARN_UNUSED_RESULT int _AesEcbDecrypt( Aes* aes, byte* out, const byte* in, word32 sz) { + int ret = 0; + #ifdef WOLF_CRYPTO_CB #ifndef WOLF_CRYPTO_CB_FIND if (aes->devId != INVALID_DEVID) #endif { - int ret = wc_CryptoCb_AesEcbDecrypt(aes, out, in, sz); + ret = wc_CryptoCb_AesEcbDecrypt(aes, out, in, sz); if (ret != CRYPTOCB_UNAVAILABLE) return ret; + ret = 0; /* fall-through when unavailable */ } #endif @@ -11188,11 +11199,12 @@ static WARN_UNUSED_RESULT int _AesEcbDecrypt( if (aes->keylen == 16) return DCPAesEcbDecrypt(aes, out, in, sz); #endif + + VECTOR_REGISTERS_PUSH; + #ifdef WOLFSSL_AESNI - if (haveAESNI && aes->use_aesni) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_ECB_decrypt(in, out, sz, (byte*)aes->key, (int)aes->rounds); - RESTORE_VECTOR_REGISTERS(); + if (aes->use_aesni) { + AES_ECB_decrypt_AESNI(in, out, sz, (byte*)aes->key, (int)aes->rounds); } else #endif @@ -11203,50 +11215,40 @@ static WARN_UNUSED_RESULT int _AesEcbDecrypt( word32 i; for (i = 0; i < sz; i += AES_BLOCK_SIZE) { - int ret = wc_AesDecryptDirect(aes, out, in); + ret = wc_AesDecryptDirect(aes, out, in); if (ret != 0) - return ret; + break; in += AES_BLOCK_SIZE; out += AES_BLOCK_SIZE; } #endif } - return 0; + VECTOR_REGISTERS_POP; + + return ret; } int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) { - int ret; - if ((in == NULL) || (out == NULL) || (aes == NULL)) return BAD_FUNC_ARG; if ((sz % AES_BLOCK_SIZE) != 0) { return BAD_LENGTH_E; } - SAVE_VECTOR_REGISTERS(return _svr_ret;); - ret = _AesEcbEncrypt(aes, out, in, sz); - RESTORE_VECTOR_REGISTERS(); - - return ret; + return _AesEcbEncrypt(aes, out, in, sz); } int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz) { - int ret; - if ((in == NULL) || (out == NULL) || (aes == NULL)) return BAD_FUNC_ARG; if ((sz % AES_BLOCK_SIZE) != 0) { return BAD_LENGTH_E; } - SAVE_VECTOR_REGISTERS(return _svr_ret;); - ret = _AesEcbDecrypt(aes, out, in, sz); - RESTORE_VECTOR_REGISTERS(); - - return ret; + return _AesEcbDecrypt(aes, out, in, sz); } #endif #endif /* HAVE_AES_ECB */ @@ -11288,7 +11290,7 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackEncrypt( in += processed; sz -= processed; - SAVE_VECTOR_REGISTERS(return _svr_ret;); + VECTOR_REGISTERS_PUSH; while (sz >= AES_BLOCK_SIZE) { /* Using aes->tmp here for inline case i.e. in=out */ @@ -11334,7 +11336,8 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackEncrypt( #endif aes->left -= sz; } - RESTORE_VECTOR_REGISTERS(); + + VECTOR_REGISTERS_POP; return ret; } @@ -11378,7 +11381,7 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackDecrypt( in += processed; sz -= processed; - SAVE_VECTOR_REGISTERS(return _svr_ret;); + VECTOR_REGISTERS_PUSH; while (sz > AES_BLOCK_SIZE) { /* Using aes->tmp here for inline case i.e. in=out */ @@ -11422,7 +11425,8 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackDecrypt( aes->left = AES_BLOCK_SIZE - sz; xorbufout(out, in, aes->tmp, sz); } - RESTORE_VECTOR_REGISTERS(); + + VECTOR_REGISTERS_POP; return ret; } @@ -11505,7 +11509,7 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackCFB8( return 0; } - SAVE_VECTOR_REGISTERS(return _svr_ret;); + VECTOR_REGISTERS_PUSH; while (sz > 0) { ret = wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg); @@ -11537,7 +11541,7 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackCFB8( sz -= 1; } - RESTORE_VECTOR_REGISTERS(); + VECTOR_REGISTERS_POP; return ret; } @@ -11561,7 +11565,7 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackCFB1( return 0; } - SAVE_VECTOR_REGISTERS(return _svr_ret;); + VECTOR_REGISTERS_PUSH; while (sz > 0) { ret = wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg); @@ -11614,7 +11618,7 @@ static WARN_UNUSED_RESULT int wc_AesFeedbackCFB1( } } - RESTORE_VECTOR_REGISTERS(); + VECTOR_REGISTERS_POP; return ret; } @@ -11792,7 +11796,7 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out, XMEMCPY(tmp, iv, KEYWRAP_BLOCK_SIZE); } - SAVE_VECTOR_REGISTERS(return _svr_ret;); + VECTOR_REGISTERS_PUSH; for (j = 0; j <= 5; j++) { for (i = 1; i <= inSz / KEYWRAP_BLOCK_SIZE; i++) { @@ -11815,7 +11819,8 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out, break; r = out + KEYWRAP_BLOCK_SIZE; } - RESTORE_VECTOR_REGISTERS(); + + VECTOR_REGISTERS_POP; if (ret != 0) return ret; @@ -11904,7 +11909,7 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out, XMEMCPY(out, in + KEYWRAP_BLOCK_SIZE, inSz - KEYWRAP_BLOCK_SIZE); XMEMSET(t, 0, sizeof(t)); - SAVE_VECTOR_REGISTERS(return _svr_ret;); + VECTOR_REGISTERS_PUSH; /* initialize counter to 6n */ n = (inSz - 1) / KEYWRAP_BLOCK_SIZE; @@ -11930,7 +11935,8 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out, if (ret != 0) break; } - RESTORE_VECTOR_REGISTERS(); + + VECTOR_REGISTERS_POP; if (ret != 0) return ret; @@ -12038,6 +12044,14 @@ int wc_AesXtsSetKey(XtsAes* aes, const byte* key, word32 len, int dir, if (ret != 0) { wc_AesFree(&aes->aes); } +#ifdef WOLFSSL_AESNI + if (aes->aes.use_aesni != aes->tweak.use_aesni) { + if (aes->aes.use_aesni) + aes->aes.use_aesni = 0; + else + aes->tweak.use_aesni = 0; + } +#endif } return ret; @@ -12124,10 +12138,10 @@ int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz, #define HAVE_INTEL_AVX2 #endif /* USE_INTEL_SPEEDUP */ -void AES_XTS_encrypt(const unsigned char *in, unsigned char *out, word32 sz, +void AES_XTS_encrypt_aesni(const unsigned char *in, unsigned char *out, word32 sz, const unsigned char* i, const unsigned char* key, const unsigned char* key2, int nr) - XASM_LINK("AES_XTS_encrypt"); + XASM_LINK("AES_XTS_encrypt_aesni"); #ifdef HAVE_INTEL_AVX1 void AES_XTS_encrypt_avx1(const unsigned char *in, unsigned char *out, word32 sz, const unsigned char* i, @@ -12137,10 +12151,10 @@ void AES_XTS_encrypt_avx1(const unsigned char *in, unsigned char *out, #endif /* HAVE_INTEL_AVX1 */ #ifdef HAVE_AES_DECRYPT -void AES_XTS_decrypt(const unsigned char *in, unsigned char *out, word32 sz, +void AES_XTS_decrypt_aesni(const unsigned char *in, unsigned char *out, word32 sz, const unsigned char* i, const unsigned char* key, const unsigned char* key2, int nr) - XASM_LINK("AES_XTS_decrypt"); + XASM_LINK("AES_XTS_decrypt_aesni"); #ifdef HAVE_INTEL_AVX1 void AES_XTS_decrypt_avx1(const unsigned char *in, unsigned char *out, word32 sz, const unsigned char* i, @@ -12217,23 +12231,17 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES * key setup passed to encrypt direct*/ - SAVE_VECTOR_REGISTERS(return _svr_ret;); - ret = wc_AesEncryptDirect(tweak, tmp, i); - if (ret != 0) { - RESTORE_VECTOR_REGISTERS(); + if (ret != 0) return ret; - } #ifdef HAVE_AES_ECB /* encrypt all of buffer at once when possible */ if (in != out) { /* can not handle inline */ XMEMCPY(out, tmp, AES_BLOCK_SIZE); - if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) { - RESTORE_VECTOR_REGISTERS(); + if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) return ret; - } } #endif @@ -12250,10 +12258,8 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, XMEMCPY(buf, in, AES_BLOCK_SIZE); xorbuf(buf, tmp, AES_BLOCK_SIZE); ret = wc_AesEncryptDirect(aes, out, buf); - if (ret != 0) { - RESTORE_VECTOR_REGISTERS(); + if (ret != 0) return ret; - } } xorbuf(out, tmp, AES_BLOCK_SIZE); @@ -12281,7 +12287,6 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, XMEMCPY(buf, out - AES_BLOCK_SIZE, AES_BLOCK_SIZE); if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */ - RESTORE_VECTOR_REGISTERS(); return BUFFER_E; } if (in != out) { @@ -12301,7 +12306,6 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, if (ret == 0) xorbuf(out - AES_BLOCK_SIZE, tmp, AES_BLOCK_SIZE); } - RESTORE_VECTOR_REGISTERS(); return ret; } @@ -12321,6 +12325,8 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, const byte* i, word32 iSz) { + int ret; + if (xaes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } @@ -12334,29 +12340,54 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, return BAD_FUNC_ARG; } + { #ifdef WOLFSSL_AESNI - #if defined(HAVE_INTEL_AVX1) - if (IS_INTEL_AVX1(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_XTS_encrypt_avx1(in, out, sz, i, (const byte*)xaes->aes.key, - (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); - RESTORE_VECTOR_REGISTERS(); - return 0; - } - else - #endif - if (haveAESNI) { - AES_XTS_encrypt(in, out, sz, i, (const byte*)xaes->aes.key, - (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); - return 0; - } - else +#ifdef WC_AES_C_DYNAMIC_FALLBACK + int orig_use_aesni = xaes->aes.use_aesni; +#endif + if (xaes->aes.use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) { +#ifdef WC_AES_C_DYNAMIC_FALLBACK + xaes->aes.use_aesni = 0; + xaes->tweak.use_aesni = 0; +#else + return ret; +#endif + } + if (xaes->aes.use_aesni) { +#if defined(HAVE_INTEL_AVX1) + if (IS_INTEL_AVX1(intel_flags)) { + AES_XTS_encrypt_avx1(in, out, sz, i, (const byte*)xaes->aes.key, + (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); + ret = 0; + } + else +#endif + { + AES_XTS_encrypt_aesni(in, out, sz, i, (const byte*)xaes->aes.key, + (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); + ret = 0; + } + } + else +#endif + { + ret = AesXtsEncrypt_sw(xaes, out, in, sz, i); + } + +#ifdef WOLFSSL_AESNI + if (xaes->aes.use_aesni) + RESTORE_VECTOR_REGISTERS(); +#ifdef WC_AES_C_DYNAMIC_FALLBACK + else if (orig_use_aesni) { + xaes->aes.use_aesni = orig_use_aesni; + xaes->tweak.use_aesni = orig_use_aesni; + } +#endif #endif - { - return AesXtsEncrypt_sw(xaes, out, in, sz, i); } -} + return ret; +} /* Same process as encryption but Aes key is AES_DECRYPTION type. * @@ -12384,13 +12415,9 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES * key setup passed to decrypt direct*/ - SAVE_VECTOR_REGISTERS(return _svr_ret;); - ret = wc_AesEncryptDirect(tweak, tmp, i); - if (ret != 0) { - RESTORE_VECTOR_REGISTERS(); + if (ret != 0) return ret; - } /* if Stealing then break out of loop one block early to handle special * case */ @@ -12402,10 +12429,8 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, /* decrypt all of buffer at once when possible */ if (in != out) { /* can not handle inline */ XMEMCPY(out, tmp, AES_BLOCK_SIZE); - if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) { - RESTORE_VECTOR_REGISTERS(); + if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) return ret; - } } #endif @@ -12419,10 +12444,8 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, XMEMCPY(buf, in, AES_BLOCK_SIZE); xorbuf(buf, tmp, AES_BLOCK_SIZE); ret = wc_AesDecryptDirect(aes, out, buf); - if (ret != 0) { - RESTORE_VECTOR_REGISTERS(); + if (ret != 0) return ret; - } } xorbuf(out, tmp, AES_BLOCK_SIZE); @@ -12465,10 +12488,8 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, XMEMCPY(buf, in, AES_BLOCK_SIZE); xorbuf(buf, tmp2, AES_BLOCK_SIZE); ret = wc_AesDecryptDirect(aes, out, buf); - if (ret != 0) { - RESTORE_VECTOR_REGISTERS(); + if (ret != 0) return ret; - } xorbuf(out, tmp2, AES_BLOCK_SIZE); /* tmp2 holds partial | last */ @@ -12480,7 +12501,6 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, /* Make buffer with end of cipher text | last */ XMEMCPY(buf, tmp2, AES_BLOCK_SIZE); if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */ - RESTORE_VECTOR_REGISTERS(); return BUFFER_E; } XMEMCPY(buf, in, sz); @@ -12488,14 +12508,11 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, xorbuf(buf, tmp, AES_BLOCK_SIZE); ret = wc_AesDecryptDirect(aes, tmp2, buf); - if (ret != 0) { - RESTORE_VECTOR_REGISTERS(); + if (ret != 0) return ret; - } xorbuf(tmp2, tmp, AES_BLOCK_SIZE); XMEMCPY(out - AES_BLOCK_SIZE, tmp2, AES_BLOCK_SIZE); } - RESTORE_VECTOR_REGISTERS(); return ret; } @@ -12515,6 +12532,8 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, const byte* i, word32 iSz) { + int ret; + if (xaes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } @@ -12528,26 +12547,49 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, return BAD_FUNC_ARG; } + { #ifdef WOLFSSL_AESNI - #if defined(HAVE_INTEL_AVX1) - if (IS_INTEL_AVX1(intel_flags)) { - SAVE_VECTOR_REGISTERS(return _svr_ret;); - AES_XTS_decrypt_avx1(in, out, sz, i, (const byte*)xaes->aes.key, - (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); - RESTORE_VECTOR_REGISTERS(); - return 0; - } - else - #endif - if (haveAESNI) { - AES_XTS_decrypt(in, out, sz, i, (const byte*)xaes->aes.key, - (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); - return 0; - } - else +#ifdef WC_AES_C_DYNAMIC_FALLBACK + int orig_use_aesni = xaes->aes.use_aesni; #endif - { - return AesXtsDecrypt_sw(xaes, out, in, sz, i); + + if (xaes->aes.use_aesni && (SAVE_VECTOR_REGISTERS2() != 0)) { + xaes->aes.use_aesni = 0; + xaes->tweak.use_aesni = 0; + } + if (xaes->aes.use_aesni) { +#if defined(HAVE_INTEL_AVX1) + if (IS_INTEL_AVX1(intel_flags)) { + AES_XTS_decrypt_avx1(in, out, sz, i, (const byte*)xaes->aes.key, + (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); + ret = 0; + } + else +#endif + { + AES_XTS_decrypt_aesni(in, out, sz, i, (const byte*)xaes->aes.key, + (const byte*)xaes->tweak.key, (int)xaes->aes.rounds); + ret = 0; + } + } + else +#endif + { + ret = AesXtsDecrypt_sw(xaes, out, in, sz, i); + } + +#ifdef WOLFSSL_AESNI + if (xaes->aes.use_aesni) + RESTORE_VECTOR_REGISTERS(); +#ifdef WC_AES_C_DYNAMIC_FALLBACK + else if (orig_use_aesni) { + xaes->aes.use_aesni = orig_use_aesni; + xaes->tweak.use_aesni = orig_use_aesni; + } +#endif +#endif + + return ret; } } #endif /* !WOLFSSL_ARMASM || WOLFSSL_ARMASM_NO_HW_CRYPTO */ diff --git a/wolfcrypt/src/aes_asm.S b/wolfcrypt/src/aes_asm.S index 89ebc7bebf..afaa0d40dd 100644 --- a/wolfcrypt/src/aes_asm.S +++ b/wolfcrypt/src/aes_asm.S @@ -30,7 +30,7 @@ #ifdef WOLFSSL_X86_64_BUILD /* -AES_CBC_encrypt (const unsigned char *in, +AES_CBC_encrypt_AESNI (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, @@ -38,11 +38,11 @@ AES_CBC_encrypt (const unsigned char *in, int nr) */ #ifndef __APPLE__ -.globl AES_CBC_encrypt -AES_CBC_encrypt: +.globl AES_CBC_encrypt_AESNI +AES_CBC_encrypt_AESNI: #else -.globl _AES_CBC_encrypt -_AES_CBC_encrypt: +.globl _AES_CBC_encrypt_AESNI +_AES_CBC_encrypt_AESNI: #endif # parameter 1: %rdi # parameter 2: %rsi @@ -95,7 +95,7 @@ ret #if defined(WOLFSSL_AESNI_BY4) /* -AES_CBC_decrypt_by4 (const unsigned char *in, +AES_CBC_decrypt_AESNI_by4 (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, @@ -103,11 +103,11 @@ AES_CBC_decrypt_by4 (const unsigned char *in, int nr) */ #ifndef __APPLE__ -.globl AES_CBC_decrypt_by4 -AES_CBC_decrypt_by4: +.globl AES_CBC_decrypt_AESNI_by4 +AES_CBC_decrypt_AESNI_by4: #else -.globl _AES_CBC_decrypt_by4 -_AES_CBC_decrypt_by4: +.globl _AES_CBC_decrypt_AESNI_by4 +_AES_CBC_decrypt_AESNI_by4: #endif # parameter 1: %rdi # parameter 2: %rsi @@ -276,7 +276,7 @@ DEND_4: #elif defined(WOLFSSL_AESNI_BY6) /* -AES_CBC_decrypt_by6 (const unsigned char *in, +AES_CBC_decrypt_AESNI_by6 (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, @@ -284,11 +284,11 @@ AES_CBC_decrypt_by6 (const unsigned char *in, int nr) */ #ifndef __APPLE__ -.globl AES_CBC_decrypt_by6 -AES_CBC_decrypt_by6: +.globl AES_CBC_decrypt_AESNI_by6 +AES_CBC_decrypt_AESNI_by6: #else -.globl _AES_CBC_decrypt_by6 -_AES_CBC_decrypt_by6: +.globl _AES_CBC_decrypt_AESNI_by6 +_AES_CBC_decrypt_AESNI_by6: #endif # parameter 1: %rdi - in # parameter 2: %rsi - out @@ -504,7 +504,7 @@ DEND_6: #else /* WOLFSSL_AESNI_BYx */ /* -AES_CBC_decrypt_by8 (const unsigned char *in, +AES_CBC_decrypt_AESNI_by8 (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, @@ -512,11 +512,11 @@ AES_CBC_decrypt_by8 (const unsigned char *in, int nr) */ #ifndef __APPLE__ -.globl AES_CBC_decrypt_by8 -AES_CBC_decrypt_by8: +.globl AES_CBC_decrypt_AESNI_by8 +AES_CBC_decrypt_AESNI_by8: #else -.globl _AES_CBC_decrypt_by8 -_AES_CBC_decrypt_by8: +.globl _AES_CBC_decrypt_AESNI_by8 +_AES_CBC_decrypt_AESNI_by8: #endif # parameter 1: %rdi - in # parameter 2: %rsi - out @@ -761,18 +761,18 @@ DEND_8: /* -AES_ECB_encrypt (const unsigned char *in, +AES_ECB_encrypt_AESNI (const unsigned char *in, unsigned char *out, unsigned long length, const unsigned char *KS, int nr) */ #ifndef __APPLE__ -.globl AES_ECB_encrypt -AES_ECB_encrypt: +.globl AES_ECB_encrypt_AESNI +AES_ECB_encrypt_AESNI: #else -.globl _AES_ECB_encrypt -_AES_ECB_encrypt: +.globl _AES_ECB_encrypt_AESNI +_AES_ECB_encrypt_AESNI: #endif # parameter 1: %rdi # parameter 2: %rsi @@ -925,18 +925,18 @@ EECB_END_4: /* -AES_ECB_decrypt (const unsigned char *in, +AES_ECB_decrypt_AESNI (const unsigned char *in, unsigned char *out, unsigned long length, const unsigned char *KS, int nr) */ #ifndef __APPLE__ -.globl AES_ECB_decrypt -AES_ECB_decrypt: +.globl AES_ECB_decrypt_AESNI +AES_ECB_decrypt_AESNI: #else -.globl _AES_ECB_decrypt -_AES_ECB_decrypt: +.globl _AES_ECB_decrypt_AESNI +_AES_ECB_decrypt_AESNI: #endif # parameter 1: %rdi # parameter 2: %rsi @@ -1092,20 +1092,19 @@ DECB_END_4: /* -void AES_128_Key_Expansion(const unsigned char* userkey, +void AES_128_Key_Expansion_AESNI(const unsigned char* userkey, unsigned char* key_schedule); */ .align 16,0x90 #ifndef __APPLE__ -.globl AES_128_Key_Expansion -AES_128_Key_Expansion: +.globl AES_128_Key_Expansion_AESNI +AES_128_Key_Expansion_AESNI: #else -.globl _AES_128_Key_Expansion -_AES_128_Key_Expansion: +.globl _AES_128_Key_Expansion_AESNI +_AES_128_Key_Expansion_AESNI: #endif # parameter 1: %rdi # parameter 2: %rsi -movl $10, 240(%rsi) movdqu (%rdi), %xmm1 movdqa %xmm1, (%rsi) @@ -1158,15 +1157,15 @@ ret /* -void AES_192_Key_Expansion (const unsigned char *userkey, +void AES_192_Key_Expansion_AESNI (const unsigned char *userkey, unsigned char *key) */ #ifndef __APPLE__ -.globl AES_192_Key_Expansion -AES_192_Key_Expansion: +.globl AES_192_Key_Expansion_AESNI +AES_192_Key_Expansion_AESNI: #else -.globl _AES_192_Key_Expansion -_AES_192_Key_Expansion: +.globl _AES_192_Key_Expansion_AESNI +_AES_192_Key_Expansion_AESNI: #endif # parameter 1: %rdi # parameter 2: %rsi @@ -1249,15 +1248,15 @@ ret /* -void AES_256_Key_Expansion (const unsigned char *userkey, +void AES_256_Key_Expansion_AESNI (const unsigned char *userkey, unsigned char *key) */ #ifndef __APPLE__ -.globl AES_256_Key_Expansion -AES_256_Key_Expansion: +.globl AES_256_Key_Expansion_AESNI +AES_256_Key_Expansion_AESNI: #else -.globl _AES_256_Key_Expansion -_AES_256_Key_Expansion: +.globl _AES_256_Key_Expansion_AESNI +_AES_256_Key_Expansion_AESNI: #endif # parameter 1: %rdi # parameter 2: %rsi @@ -1337,7 +1336,7 @@ ret #elif defined WOLFSSL_X86_BUILD /* -AES_CBC_encrypt (const unsigned char *in, +AES_CBC_encrypt_AESNI (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, @@ -1345,11 +1344,11 @@ AES_CBC_encrypt (const unsigned char *in, int nr) */ #ifndef __APPLE__ -.globl AES_CBC_encrypt -AES_CBC_encrypt: +.globl AES_CBC_encrypt_AESNI +AES_CBC_encrypt_AESNI: #else -.globl _AES_CBC_encrypt -_AES_CBC_encrypt: +.globl _AES_CBC_encrypt_AESNI +_AES_CBC_encrypt_AESNI: #endif # parameter 1: stack[4] => %edi # parameter 2: stack[8] => %esi @@ -1416,7 +1415,7 @@ _AES_CBC_encrypt: /* -AES_CBC_decrypt_by4 (const unsigned char *in, +AES_CBC_decrypt_AESNI_by4 (const unsigned char *in, unsigned char *out, unsigned char ivec[16], unsigned long length, @@ -1424,11 +1423,11 @@ AES_CBC_decrypt_by4 (const unsigned char *in, int nr) */ #ifndef __APPLE__ -.globl AES_CBC_decrypt_by4 -AES_CBC_decrypt_by4: +.globl AES_CBC_decrypt_AESNI_by4 +AES_CBC_decrypt_AESNI_by4: #else -.globl _AES_CBC_decrypt_by4 -_AES_CBC_decrypt_by4: +.globl _AES_CBC_decrypt_AESNI_by4 +_AES_CBC_decrypt_AESNI_by4: #endif # parameter 1: stack[4] => %edi # parameter 2: stack[8] => %esi @@ -1614,18 +1613,18 @@ DEND_4: ret /* -AES_ECB_encrypt (const unsigned char *in, +AES_ECB_encrypt_AESNI (const unsigned char *in, unsigned char *out, unsigned long length, const unsigned char *KS, int nr) */ #ifndef __APPLE__ -.globl AES_ECB_encrypt -AES_ECB_encrypt: +.globl AES_ECB_encrypt_AESNI +AES_ECB_encrypt_AESNI: #else -.globl _AES_ECB_encrypt -_AES_ECB_encrypt: +.globl _AES_ECB_encrypt_AESNI +_AES_ECB_encrypt_AESNI: #endif # parameter 1: stack[4] => %edi # parameter 2: stack[8] => %esi @@ -1791,18 +1790,18 @@ EECB_END_4: /* -AES_ECB_decrypt (const unsigned char *in, +AES_ECB_decrypt_AESNI (const unsigned char *in, unsigned char *out, unsigned long length, const unsigned char *KS, int nr) */ #ifndef __APPLE__ -.globl AES_ECB_decrypt -AES_ECB_decrypt: +.globl AES_ECB_decrypt_AESNI +AES_ECB_decrypt_AESNI: #else -.globl _AES_ECB_decrypt -_AES_ECB_decrypt: +.globl _AES_ECB_decrypt_AESNI +_AES_ECB_decrypt_AESNI: #endif # parameter 1: stack[4] => %edi # parameter 2: stack[8] => %esi @@ -1969,16 +1968,16 @@ DECB_END_4: /* -void AES_128_Key_Expansion(const unsigned char* userkey, +void AES_128_Key_Expansion_AESNI(const unsigned char* userkey, unsigned char* key_schedule); */ .align 16,0x90 #ifndef __APPLE__ -.globl AES_128_Key_Expansion -AES_128_Key_Expansion: +.globl AES_128_Key_Expansion_AESNI +AES_128_Key_Expansion_AESNI: #else -.globl _AES_128_Key_Expansion -_AES_128_Key_Expansion: +.globl _AES_128_Key_Expansion_AESNI +_AES_128_Key_Expansion_AESNI: #endif # parameter 1: stack[4] => %eax # parameter 2: stack[8] => %edx @@ -2038,15 +2037,15 @@ PREPARE_ROUNDKEY_128: /* -void AES_192_Key_Expansion (const unsigned char *userkey, +void AES_192_Key_Expansion_AESNI (const unsigned char *userkey, unsigned char *key) */ #ifndef __APPLE__ -.globl AES_192_Key_Expansion -AES_192_Key_Expansion: +.globl AES_192_Key_Expansion_AESNI +AES_192_Key_Expansion_AESNI: #else -.globl _AES_192_Key_Expansion -_AES_192_Key_Expansion: +.globl _AES_192_Key_Expansion_AESNI +_AES_192_Key_Expansion_AESNI: #endif # parameter 1: stack[4] => %eax # parameter 2: stack[8] => %edx @@ -2131,15 +2130,15 @@ PREPARE_ROUNDKEY_192: /* -void AES_256_Key_Expansion (const unsigned char *userkey, +void AES_256_Key_Expansion_AESNI (const unsigned char *userkey, unsigned char *key) */ #ifndef __APPLE__ -.globl AES_256_Key_Expansion -AES_256_Key_Expansion: +.globl AES_256_Key_Expansion_AESNI +AES_256_Key_Expansion_AESNI: #else -.globl _AES_256_Key_Expansion -_AES_256_Key_Expansion: +.globl _AES_256_Key_Expansion_AESNI +_AES_256_Key_Expansion_AESNI: #endif # parameter 1: stack[4] => %eax # parameter 2: stack[8] => %edx diff --git a/wolfcrypt/src/aes_asm.asm b/wolfcrypt/src/aes_asm.asm index aa2fa4b412..000789b402 100644 --- a/wolfcrypt/src/aes_asm.asm +++ b/wolfcrypt/src/aes_asm.asm @@ -47,14 +47,14 @@ ENDIF ; /* -; AES_CBC_encrypt[const ,unsigned char*in +; AES_CBC_encrypt_AESNI[const ,unsigned char*in ; unsigned ,char*out ; unsigned ,char ivec+16 ; unsigned ,long length ; const ,unsigned char*KS ; int nr] ; */ -AES_CBC_encrypt PROC +AES_CBC_encrypt_AESNI PROC ;# parameter 1: rdi ;# parameter 2: rsi ;# parameter 3: rdx @@ -117,16 +117,16 @@ LAST: mov rdi,rax mov rsi,r11 ret -AES_CBC_encrypt ENDP +AES_CBC_encrypt_AESNI ENDP -; void AES_CBC_decrypt_by4(const unsigned char* in, +; void AES_CBC_decrypt_AESNI_by4(const unsigned char* in, ; unsigned char* out, ; unsigned char ivec[16], ; unsigned long length, ; const unsigned char* KS, ; int nr) -AES_CBC_decrypt_by4 PROC +AES_CBC_decrypt_AESNI_by4 PROC ; parameter 1: rdi ; parameter 2: rsi ; parameter 3: rdx @@ -325,16 +325,16 @@ DEND_4: movdqa xmm15, [rsp+112] add rsp, 8+8*16 ; 8 = align stack , 8 xmm6-12,15 16 bytes each ret -AES_CBC_decrypt_by4 ENDP +AES_CBC_decrypt_AESNI_by4 ENDP -; void AES_CBC_decrypt_by6(const unsigned char *in, +; void AES_CBC_decrypt_AESNI_by6(const unsigned char *in, ; unsigned char *out, ; unsigned char ivec[16], ; unsigned long length, ; const unsigned char *KS, ; int nr) -AES_CBC_decrypt_by6 PROC +AES_CBC_decrypt_AESNI_by6 PROC ; parameter 1: rdi - in ; parameter 2: rsi - out ; parameter 3: rdx - ivec @@ -582,16 +582,16 @@ DEND_6: movdqa xmm14, [rsp+128] add rsp, 8+9*16 ; 8 = align stack , 9 xmm6-14 16 bytes each ret -AES_CBC_decrypt_by6 ENDP +AES_CBC_decrypt_AESNI_by6 ENDP -; void AES_CBC_decrypt_by8(const unsigned char *in, +; void AES_CBC_decrypt_AESNI_by8(const unsigned char *in, ; unsigned char *out, ; unsigned char ivec[16], ; unsigned long length, ; const unsigned char *KS, ; int nr) -AES_CBC_decrypt_by8 PROC +AES_CBC_decrypt_AESNI_by8 PROC ; parameter 1: rdi - in ; parameter 2: rsi - out ; parameter 3: rdx - ivec @@ -865,18 +865,18 @@ DEND_8: movdqa xmm13, [rsp+112] add rsp, 8+8*16 ; 8 = align stack , 8 xmm6-13 16 bytes each ret -AES_CBC_decrypt_by8 ENDP +AES_CBC_decrypt_AESNI_by8 ENDP ; /* -; AES_ECB_encrypt[const ,unsigned char*in +; AES_ECB_encrypt_AESNI[const ,unsigned char*in ; unsigned ,char*out ; unsigned ,long length ; const ,unsigned char*KS ; int nr] ; */ ; . globl AES_ECB_encrypt -AES_ECB_encrypt PROC +AES_ECB_encrypt_AESNI PROC ;# parameter 1: rdi ;# parameter 2: rsi ;# parameter 3: rdx @@ -1054,7 +1054,7 @@ EECB_END_4: movdqa xmm12, [rsp+48] add rsp,8+4*16 ; 8 = align stack , 4 xmm9-12 16 bytes each ret -AES_ECB_encrypt ENDP +AES_ECB_encrypt_AESNI ENDP ; /* ; AES_ECB_decrypt[const ,unsigned char*in @@ -1241,17 +1241,17 @@ DECB_END_4: movdqa xmm12, [rsp+48] add rsp,8+4*16 ; 8 = align stack , 4 xmm9-12 16 bytes each ret -AES_ECB_decrypt ENDP +AES_ECB_decrypt_AESNI ENDP ; /* -; void ,AES_128_Key_Expansion[const unsigned char*userkey +; void ,AES_128_Key_Expansion_AESNI[const unsigned char*userkey ; unsigned char*key_schedule]/ ; */ ; . align 16,0x90 ; . globl AES_128_Key_Expansion -AES_128_Key_Expansion PROC +AES_128_Key_Expansion_AESNI PROC ;# parameter 1: rdi ;# parameter 2: rsi @@ -1322,14 +1322,14 @@ PREPARE_ROUNDKEY_128: pxor xmm1,xmm3 pxor xmm1,xmm2 ret -AES_128_Key_Expansion ENDP +AES_128_Key_Expansion_AESNI ENDP ; /* -; void ,AES_192_Key_Expansion[const unsigned char*userkey +; void ,AES_192_Key_Expansion_AESNI[const unsigned char*userkey ; unsigned char*key] ; */ ; . globl AES_192_Key_Expansion -AES_192_Key_Expansion PROC +AES_192_Key_Expansion_AESNI PROC ;# parameter 1: rdi ;# parameter 2: rsi @@ -1426,14 +1426,14 @@ PREPARE_ROUNDKEY_192: pxor xmm3,xmm4 pxor xmm3,xmm2 ret -AES_192_Key_Expansion ENDP +AES_192_Key_Expansion_AESNI ENDP ; /* -; void ,AES_256_Key_Expansion[const unsigned char*userkey +; void ,AES_256_Key_Expansion_AESNI[const unsigned char*userkey ; unsigned char*key] ; */ ; . globl AES_256_Key_Expansion -AES_256_Key_Expansion PROC +AES_256_Key_Expansion_AESNI PROC ;# parameter 1: rdi ;# parameter 2: rsi @@ -1495,7 +1495,7 @@ AES_256_Key_Expansion PROC mov rdi,rax mov rsi,r11 ret -AES_256_Key_Expansion ENDP +AES_256_Key_Expansion_AESNI ENDP MAKE_RK256_a: pshufd xmm2,xmm2,0ffh diff --git a/wolfcrypt/src/aes_gcm_asm.S b/wolfcrypt/src/aes_gcm_asm.S index b17afcf3e9..4175888f12 100644 --- a/wolfcrypt/src/aes_gcm_asm.S +++ b/wolfcrypt/src/aes_gcm_asm.S @@ -180,15 +180,15 @@ L_aes_gcm_mod2_128: .quad 0x1, 0xc200000000000000 #ifndef __APPLE__ .text -.globl AES_GCM_encrypt -.type AES_GCM_encrypt,@function +.globl AES_GCM_encrypt_aesni +.type AES_GCM_encrypt_aesni,@function .align 16 -AES_GCM_encrypt: +AES_GCM_encrypt_aesni: #else .section __TEXT,__text -.globl _AES_GCM_encrypt +.globl _AES_GCM_encrypt_aesni .p2align 4 -_AES_GCM_encrypt: +_AES_GCM_encrypt_aesni: #endif /* __APPLE__ */ pushq %r13 pushq %r12 @@ -207,7 +207,7 @@ _AES_GCM_encrypt: pxor %xmm6, %xmm6 cmpl $12, %ebx movl %ebx, %edx - jne L_AES_GCM_encrypt_iv_not_12 + jne L_AES_GCM_encrypt_aesni_iv_not_12 # # Calculate values when IV is 12 bytes # Set counter based on IV movl $0x1000000, %ecx @@ -247,7 +247,7 @@ _AES_GCM_encrypt: aesenc %xmm7, %xmm1 cmpl $11, %r10d movdqa 160(%r15), %xmm7 - jl L_AES_GCM_encrypt_calc_iv_12_last + jl L_AES_GCM_encrypt_aesni_calc_iv_12_last aesenc %xmm7, %xmm5 aesenc %xmm7, %xmm1 movdqa 176(%r15), %xmm7 @@ -255,20 +255,20 @@ _AES_GCM_encrypt: aesenc %xmm7, %xmm1 cmpl $13, %r10d movdqa 192(%r15), %xmm7 - jl L_AES_GCM_encrypt_calc_iv_12_last + jl L_AES_GCM_encrypt_aesni_calc_iv_12_last aesenc %xmm7, %xmm5 aesenc %xmm7, %xmm1 movdqa 208(%r15), %xmm7 aesenc %xmm7, %xmm5 aesenc %xmm7, %xmm1 movdqa 224(%r15), %xmm7 -L_AES_GCM_encrypt_calc_iv_12_last: +L_AES_GCM_encrypt_aesni_calc_iv_12_last: aesenclast %xmm7, %xmm5 aesenclast %xmm7, %xmm1 pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 movdqu %xmm1, 144(%rsp) - jmp L_AES_GCM_encrypt_iv_done -L_AES_GCM_encrypt_iv_not_12: + jmp L_AES_GCM_encrypt_aesni_iv_done +L_AES_GCM_encrypt_aesni_iv_not_12: # Calculate values when IV is not 12 bytes # H = Encrypt X(=0) movdqa (%r15), %xmm5 @@ -283,27 +283,27 @@ L_AES_GCM_encrypt_iv_not_12: aesenc 144(%r15), %xmm5 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last aesenc %xmm9, %xmm5 aesenc 176(%r15), %xmm5 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last aesenc %xmm9, %xmm5 aesenc 208(%r15), %xmm5 movdqa 224(%r15), %xmm9 -L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last: aesenclast %xmm9, %xmm5 pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 # Calc counter # Initialization vector cmpl $0x00, %edx movq $0x00, %rcx - je L_AES_GCM_encrypt_calc_iv_done + je L_AES_GCM_encrypt_aesni_calc_iv_done cmpl $16, %edx - jl L_AES_GCM_encrypt_calc_iv_lt16 + jl L_AES_GCM_encrypt_aesni_calc_iv_lt16 andl $0xfffffff0, %edx -L_AES_GCM_encrypt_calc_iv_16_loop: +L_AES_GCM_encrypt_aesni_calc_iv_16_loop: movdqu (%rax,%rcx,1), %xmm8 pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 pxor %xmm8, %xmm4 @@ -363,22 +363,22 @@ L_AES_GCM_encrypt_calc_iv_16_loop: pxor %xmm2, %xmm4 addl $16, %ecx cmpl %edx, %ecx - jl L_AES_GCM_encrypt_calc_iv_16_loop + jl L_AES_GCM_encrypt_aesni_calc_iv_16_loop movl %ebx, %edx cmpl %edx, %ecx - je L_AES_GCM_encrypt_calc_iv_done -L_AES_GCM_encrypt_calc_iv_lt16: + je L_AES_GCM_encrypt_aesni_calc_iv_done +L_AES_GCM_encrypt_aesni_calc_iv_lt16: subq $16, %rsp pxor %xmm8, %xmm8 xorl %ebx, %ebx movdqu %xmm8, (%rsp) -L_AES_GCM_encrypt_calc_iv_loop: +L_AES_GCM_encrypt_aesni_calc_iv_loop: movzbl (%rax,%rcx,1), %r13d movb %r13b, (%rsp,%rbx,1) incl %ecx incl %ebx cmpl %edx, %ecx - jl L_AES_GCM_encrypt_calc_iv_loop + jl L_AES_GCM_encrypt_aesni_calc_iv_loop movdqu (%rsp), %xmm8 addq $16, %rsp pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 @@ -437,7 +437,7 @@ L_AES_GCM_encrypt_calc_iv_loop: pxor %xmm1, %xmm2 pxor %xmm7, %xmm2 pxor %xmm2, %xmm4 -L_AES_GCM_encrypt_calc_iv_done: +L_AES_GCM_encrypt_aesni_calc_iv_done: # T = Encrypt counter pxor %xmm0, %xmm0 shll $3, %edx @@ -512,28 +512,28 @@ L_AES_GCM_encrypt_calc_iv_done: aesenc 144(%r15), %xmm8 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last aesenc %xmm9, %xmm8 aesenc 176(%r15), %xmm8 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last aesenc %xmm9, %xmm8 aesenc 208(%r15), %xmm8 movdqa 224(%r15), %xmm9 -L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last: aesenclast %xmm9, %xmm8 movdqu %xmm8, 144(%rsp) -L_AES_GCM_encrypt_iv_done: +L_AES_GCM_encrypt_aesni_iv_done: # Additional authentication data movl %r11d, %edx cmpl $0x00, %edx - je L_AES_GCM_encrypt_calc_aad_done + je L_AES_GCM_encrypt_aesni_calc_aad_done xorl %ecx, %ecx cmpl $16, %edx - jl L_AES_GCM_encrypt_calc_aad_lt16 + jl L_AES_GCM_encrypt_aesni_calc_aad_lt16 andl $0xfffffff0, %edx -L_AES_GCM_encrypt_calc_aad_16_loop: +L_AES_GCM_encrypt_aesni_calc_aad_16_loop: movdqu (%r12,%rcx,1), %xmm8 pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 pxor %xmm8, %xmm6 @@ -593,22 +593,22 @@ L_AES_GCM_encrypt_calc_aad_16_loop: pxor %xmm2, %xmm6 addl $16, %ecx cmpl %edx, %ecx - jl L_AES_GCM_encrypt_calc_aad_16_loop + jl L_AES_GCM_encrypt_aesni_calc_aad_16_loop movl %r11d, %edx cmpl %edx, %ecx - je L_AES_GCM_encrypt_calc_aad_done -L_AES_GCM_encrypt_calc_aad_lt16: + je L_AES_GCM_encrypt_aesni_calc_aad_done +L_AES_GCM_encrypt_aesni_calc_aad_lt16: subq $16, %rsp pxor %xmm8, %xmm8 xorl %ebx, %ebx movdqu %xmm8, (%rsp) -L_AES_GCM_encrypt_calc_aad_loop: +L_AES_GCM_encrypt_aesni_calc_aad_loop: movzbl (%r12,%rcx,1), %r13d movb %r13b, (%rsp,%rbx,1) incl %ecx incl %ebx cmpl %edx, %ecx - jl L_AES_GCM_encrypt_calc_aad_loop + jl L_AES_GCM_encrypt_aesni_calc_aad_loop movdqu (%rsp), %xmm8 addq $16, %rsp pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 @@ -667,7 +667,7 @@ L_AES_GCM_encrypt_calc_aad_loop: pxor %xmm1, %xmm2 pxor %xmm7, %xmm2 pxor %xmm2, %xmm6 -L_AES_GCM_encrypt_calc_aad_done: +L_AES_GCM_encrypt_aesni_calc_aad_done: # Calculate counter and H pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 movdqa %xmm5, %xmm9 @@ -685,7 +685,7 @@ L_AES_GCM_encrypt_calc_aad_done: xorq %rbx, %rbx cmpl $0x80, %r9d movl %r9d, %r13d - jl L_AES_GCM_encrypt_done_128 + jl L_AES_GCM_encrypt_aesni_done_128 andl $0xffffff80, %r13d movdqa %xmm6, %xmm2 # H ^ 1 @@ -1104,7 +1104,7 @@ L_AES_GCM_encrypt_calc_aad_done: aesenc %xmm7, %xmm15 cmpl $11, %r10d movdqa 160(%r15), %xmm7 - jl L_AES_GCM_encrypt_enc_done + jl L_AES_GCM_encrypt_aesni_enc_done aesenc %xmm7, %xmm8 aesenc %xmm7, %xmm9 aesenc %xmm7, %xmm10 @@ -1124,7 +1124,7 @@ L_AES_GCM_encrypt_calc_aad_done: aesenc %xmm7, %xmm15 cmpl $13, %r10d movdqa 192(%r15), %xmm7 - jl L_AES_GCM_encrypt_enc_done + jl L_AES_GCM_encrypt_aesni_enc_done aesenc %xmm7, %xmm8 aesenc %xmm7, %xmm9 aesenc %xmm7, %xmm10 @@ -1143,7 +1143,7 @@ L_AES_GCM_encrypt_calc_aad_done: aesenc %xmm7, %xmm14 aesenc %xmm7, %xmm15 movdqa 224(%r15), %xmm7 -L_AES_GCM_encrypt_enc_done: +L_AES_GCM_encrypt_aesni_enc_done: aesenclast %xmm7, %xmm8 aesenclast %xmm7, %xmm9 movdqu (%rdi), %xmm0 @@ -1178,9 +1178,9 @@ L_AES_GCM_encrypt_enc_done: movdqu %xmm15, 112(%rsi) cmpl $0x80, %r13d movl $0x80, %ebx - jle L_AES_GCM_encrypt_end_128 + jle L_AES_GCM_encrypt_aesni_end_128 # More 128 bytes of input -L_AES_GCM_encrypt_ghash_128: +L_AES_GCM_encrypt_aesni_ghash_128: leaq (%rdi,%rbx,1), %rcx leaq (%rsi,%rbx,1), %rdx movdqu 128(%rsp), %xmm8 @@ -1448,7 +1448,7 @@ L_AES_GCM_encrypt_ghash_128: pxor %xmm3, %xmm2 cmpl $11, %r10d movdqa 160(%r15), %xmm7 - jl L_AES_GCM_encrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done aesenc %xmm7, %xmm8 aesenc %xmm7, %xmm9 aesenc %xmm7, %xmm10 @@ -1468,7 +1468,7 @@ L_AES_GCM_encrypt_ghash_128: aesenc %xmm7, %xmm15 cmpl $13, %r10d movdqa 192(%r15), %xmm7 - jl L_AES_GCM_encrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done aesenc %xmm7, %xmm8 aesenc %xmm7, %xmm9 aesenc %xmm7, %xmm10 @@ -1487,7 +1487,7 @@ L_AES_GCM_encrypt_ghash_128: aesenc %xmm7, %xmm14 aesenc %xmm7, %xmm15 movdqa 224(%r15), %xmm7 -L_AES_GCM_encrypt_aesenc_128_ghash_avx_done: +L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done: aesenclast %xmm7, %xmm8 aesenclast %xmm7, %xmm9 movdqu (%rcx), %xmm0 @@ -1522,8 +1522,8 @@ L_AES_GCM_encrypt_aesenc_128_ghash_avx_done: movdqu %xmm15, 112(%rdx) addl $0x80, %ebx cmpl %r13d, %ebx - jl L_AES_GCM_encrypt_ghash_128 -L_AES_GCM_encrypt_end_128: + jl L_AES_GCM_encrypt_aesni_ghash_128 +L_AES_GCM_encrypt_aesni_end_128: movdqa L_aes_gcm_bswap_mask(%rip), %xmm4 pshufb %xmm4, %xmm8 pshufb %xmm4, %xmm9 @@ -1710,14 +1710,14 @@ L_AES_GCM_encrypt_end_128: pxor %xmm4, %xmm2 pxor %xmm2, %xmm6 movdqu (%rsp), %xmm5 -L_AES_GCM_encrypt_done_128: +L_AES_GCM_encrypt_aesni_done_128: movl %r9d, %edx cmpl %edx, %ebx - jge L_AES_GCM_encrypt_done_enc + jge L_AES_GCM_encrypt_aesni_done_enc movl %r9d, %r13d andl $0xfffffff0, %r13d cmpl %r13d, %ebx - jge L_AES_GCM_encrypt_last_block_done + jge L_AES_GCM_encrypt_aesni_last_block_done leaq (%rdi,%rbx,1), %rcx leaq (%rsi,%rbx,1), %rdx movdqu 128(%rsp), %xmm8 @@ -1737,16 +1737,16 @@ L_AES_GCM_encrypt_done_128: aesenc 144(%r15), %xmm8 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last aesenc %xmm9, %xmm8 aesenc 176(%r15), %xmm8 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last aesenc %xmm9, %xmm8 aesenc 208(%r15), %xmm8 movdqa 224(%r15), %xmm9 -L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last: aesenclast %xmm9, %xmm8 movdqu (%rcx), %xmm9 pxor %xmm9, %xmm8 @@ -1755,8 +1755,8 @@ L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last: pxor %xmm8, %xmm6 addl $16, %ebx cmpl %r13d, %ebx - jge L_AES_GCM_encrypt_last_block_ghash -L_AES_GCM_encrypt_last_block_start: + jge L_AES_GCM_encrypt_aesni_last_block_ghash +L_AES_GCM_encrypt_aesni_last_block_start: leaq (%rdi,%rbx,1), %rcx leaq (%rsi,%rbx,1), %rdx movdqu 128(%rsp), %xmm8 @@ -1801,16 +1801,16 @@ L_AES_GCM_encrypt_last_block_start: pxor %xmm3, %xmm6 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_encrypt_aesenc_gfmul_last + jl L_AES_GCM_encrypt_aesni_aesenc_gfmul_last aesenc %xmm9, %xmm8 aesenc 176(%r15), %xmm8 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_encrypt_aesenc_gfmul_last + jl L_AES_GCM_encrypt_aesni_aesenc_gfmul_last aesenc %xmm9, %xmm8 aesenc 208(%r15), %xmm8 movdqa 224(%r15), %xmm9 -L_AES_GCM_encrypt_aesenc_gfmul_last: +L_AES_GCM_encrypt_aesni_aesenc_gfmul_last: aesenclast %xmm9, %xmm8 movdqu (%rcx), %xmm9 pxor %xmm9, %xmm8 @@ -1819,8 +1819,8 @@ L_AES_GCM_encrypt_aesenc_gfmul_last: pxor %xmm8, %xmm6 addl $16, %ebx cmpl %r13d, %ebx - jl L_AES_GCM_encrypt_last_block_start -L_AES_GCM_encrypt_last_block_ghash: + jl L_AES_GCM_encrypt_aesni_last_block_start +L_AES_GCM_encrypt_aesni_last_block_ghash: pshufd $0x4e, %xmm5, %xmm9 pshufd $0x4e, %xmm6, %xmm10 movdqa %xmm6, %xmm11 @@ -1861,11 +1861,11 @@ L_AES_GCM_encrypt_last_block_ghash: pxor %xmm13, %xmm14 pxor %xmm8, %xmm14 pxor %xmm14, %xmm6 -L_AES_GCM_encrypt_last_block_done: +L_AES_GCM_encrypt_aesni_last_block_done: movl %r9d, %ecx movl %ecx, %edx andl $15, %ecx - jz L_AES_GCM_encrypt_aesenc_last15_enc_avx_done + jz L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done movdqu 128(%rsp), %xmm4 pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 pxor (%r15), %xmm4 @@ -1880,21 +1880,21 @@ L_AES_GCM_encrypt_last_block_done: aesenc 144(%r15), %xmm4 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last aesenc %xmm9, %xmm4 aesenc 176(%r15), %xmm4 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last aesenc %xmm9, %xmm4 aesenc 208(%r15), %xmm4 movdqa 224(%r15), %xmm9 -L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last: aesenclast %xmm9, %xmm4 subq $16, %rsp xorl %ecx, %ecx movdqu %xmm4, (%rsp) -L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop: +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop: movzbl (%rdi,%rbx,1), %r13d xorb (%rsp,%rcx,1), %r13b movb %r13b, (%rsi,%rbx,1) @@ -1902,16 +1902,16 @@ L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop: incl %ebx incl %ecx cmpl %edx, %ebx - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop xorq %r13, %r13 cmpl $16, %ecx - je L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc -L_AES_GCM_encrypt_aesenc_last15_enc_avx_byte_loop: + je L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop: movb %r13b, (%rsp,%rcx,1) incl %ecx cmpl $16, %ecx - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_byte_loop -L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc: + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc: movdqu (%rsp), %xmm4 addq $16, %rsp pshufb L_aes_gcm_bswap_mask(%rip), %xmm4 @@ -1956,8 +1956,8 @@ L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc: pxor %xmm13, %xmm14 pxor %xmm8, %xmm14 pxor %xmm14, %xmm6 -L_AES_GCM_encrypt_aesenc_last15_enc_avx_done: -L_AES_GCM_encrypt_done_enc: +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_aesni_done_enc: movl %r9d, %edx movl %r11d, %ecx shlq $3, %rdx @@ -2009,19 +2009,19 @@ L_AES_GCM_encrypt_done_enc: movdqu 144(%rsp), %xmm0 pxor %xmm6, %xmm0 cmpl $16, %r14d - je L_AES_GCM_encrypt_store_tag_16 + je L_AES_GCM_encrypt_aesni_store_tag_16 xorq %rcx, %rcx movdqu %xmm0, (%rsp) -L_AES_GCM_encrypt_store_tag_loop: +L_AES_GCM_encrypt_aesni_store_tag_loop: movzbl (%rsp,%rcx,1), %r13d movb %r13b, (%r8,%rcx,1) incl %ecx cmpl %r14d, %ecx - jne L_AES_GCM_encrypt_store_tag_loop - jmp L_AES_GCM_encrypt_store_tag_done -L_AES_GCM_encrypt_store_tag_16: + jne L_AES_GCM_encrypt_aesni_store_tag_loop + jmp L_AES_GCM_encrypt_aesni_store_tag_done +L_AES_GCM_encrypt_aesni_store_tag_16: movdqu %xmm0, (%r8) -L_AES_GCM_encrypt_store_tag_done: +L_AES_GCM_encrypt_aesni_store_tag_done: addq $0xa0, %rsp popq %r15 popq %r14 @@ -2030,19 +2030,19 @@ L_AES_GCM_encrypt_store_tag_done: popq %r13 repz retq #ifndef __APPLE__ -.size AES_GCM_encrypt,.-AES_GCM_encrypt +.size AES_GCM_encrypt_aesni,.-AES_GCM_encrypt_aesni #endif /* __APPLE__ */ #ifndef __APPLE__ .text -.globl AES_GCM_decrypt -.type AES_GCM_decrypt,@function +.globl AES_GCM_decrypt_aesni +.type AES_GCM_decrypt_aesni,@function .align 16 -AES_GCM_decrypt: +AES_GCM_decrypt_aesni: #else .section __TEXT,__text -.globl _AES_GCM_decrypt +.globl _AES_GCM_decrypt_aesni .p2align 4 -_AES_GCM_decrypt: +_AES_GCM_decrypt_aesni: #endif /* __APPLE__ */ pushq %r13 pushq %r12 @@ -2063,7 +2063,7 @@ _AES_GCM_decrypt: pxor %xmm6, %xmm6 cmpl $12, %ebx movl %ebx, %edx - jne L_AES_GCM_decrypt_iv_not_12 + jne L_AES_GCM_decrypt_aesni_iv_not_12 # # Calculate values when IV is 12 bytes # Set counter based on IV movl $0x1000000, %ecx @@ -2103,7 +2103,7 @@ _AES_GCM_decrypt: aesenc %xmm7, %xmm1 cmpl $11, %r10d movdqa 160(%r15), %xmm7 - jl L_AES_GCM_decrypt_calc_iv_12_last + jl L_AES_GCM_decrypt_aesni_calc_iv_12_last aesenc %xmm7, %xmm5 aesenc %xmm7, %xmm1 movdqa 176(%r15), %xmm7 @@ -2111,20 +2111,20 @@ _AES_GCM_decrypt: aesenc %xmm7, %xmm1 cmpl $13, %r10d movdqa 192(%r15), %xmm7 - jl L_AES_GCM_decrypt_calc_iv_12_last + jl L_AES_GCM_decrypt_aesni_calc_iv_12_last aesenc %xmm7, %xmm5 aesenc %xmm7, %xmm1 movdqa 208(%r15), %xmm7 aesenc %xmm7, %xmm5 aesenc %xmm7, %xmm1 movdqa 224(%r15), %xmm7 -L_AES_GCM_decrypt_calc_iv_12_last: +L_AES_GCM_decrypt_aesni_calc_iv_12_last: aesenclast %xmm7, %xmm5 aesenclast %xmm7, %xmm1 pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 movdqu %xmm1, 144(%rsp) - jmp L_AES_GCM_decrypt_iv_done -L_AES_GCM_decrypt_iv_not_12: + jmp L_AES_GCM_decrypt_aesni_iv_done +L_AES_GCM_decrypt_aesni_iv_not_12: # Calculate values when IV is not 12 bytes # H = Encrypt X(=0) movdqa (%r15), %xmm5 @@ -2139,27 +2139,27 @@ L_AES_GCM_decrypt_iv_not_12: aesenc 144(%r15), %xmm5 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last aesenc %xmm9, %xmm5 aesenc 176(%r15), %xmm5 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last aesenc %xmm9, %xmm5 aesenc 208(%r15), %xmm5 movdqa 224(%r15), %xmm9 -L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last: +L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last: aesenclast %xmm9, %xmm5 pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 # Calc counter # Initialization vector cmpl $0x00, %edx movq $0x00, %rcx - je L_AES_GCM_decrypt_calc_iv_done + je L_AES_GCM_decrypt_aesni_calc_iv_done cmpl $16, %edx - jl L_AES_GCM_decrypt_calc_iv_lt16 + jl L_AES_GCM_decrypt_aesni_calc_iv_lt16 andl $0xfffffff0, %edx -L_AES_GCM_decrypt_calc_iv_16_loop: +L_AES_GCM_decrypt_aesni_calc_iv_16_loop: movdqu (%rax,%rcx,1), %xmm8 pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 pxor %xmm8, %xmm4 @@ -2219,22 +2219,22 @@ L_AES_GCM_decrypt_calc_iv_16_loop: pxor %xmm2, %xmm4 addl $16, %ecx cmpl %edx, %ecx - jl L_AES_GCM_decrypt_calc_iv_16_loop + jl L_AES_GCM_decrypt_aesni_calc_iv_16_loop movl %ebx, %edx cmpl %edx, %ecx - je L_AES_GCM_decrypt_calc_iv_done -L_AES_GCM_decrypt_calc_iv_lt16: + je L_AES_GCM_decrypt_aesni_calc_iv_done +L_AES_GCM_decrypt_aesni_calc_iv_lt16: subq $16, %rsp pxor %xmm8, %xmm8 xorl %ebx, %ebx movdqu %xmm8, (%rsp) -L_AES_GCM_decrypt_calc_iv_loop: +L_AES_GCM_decrypt_aesni_calc_iv_loop: movzbl (%rax,%rcx,1), %r13d movb %r13b, (%rsp,%rbx,1) incl %ecx incl %ebx cmpl %edx, %ecx - jl L_AES_GCM_decrypt_calc_iv_loop + jl L_AES_GCM_decrypt_aesni_calc_iv_loop movdqu (%rsp), %xmm8 addq $16, %rsp pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 @@ -2293,7 +2293,7 @@ L_AES_GCM_decrypt_calc_iv_loop: pxor %xmm1, %xmm2 pxor %xmm7, %xmm2 pxor %xmm2, %xmm4 -L_AES_GCM_decrypt_calc_iv_done: +L_AES_GCM_decrypt_aesni_calc_iv_done: # T = Encrypt counter pxor %xmm0, %xmm0 shll $3, %edx @@ -2368,28 +2368,28 @@ L_AES_GCM_decrypt_calc_iv_done: aesenc 144(%r15), %xmm8 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last aesenc %xmm9, %xmm8 aesenc 176(%r15), %xmm8 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last aesenc %xmm9, %xmm8 aesenc 208(%r15), %xmm8 movdqa 224(%r15), %xmm9 -L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last: +L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last: aesenclast %xmm9, %xmm8 movdqu %xmm8, 144(%rsp) -L_AES_GCM_decrypt_iv_done: +L_AES_GCM_decrypt_aesni_iv_done: # Additional authentication data movl %r11d, %edx cmpl $0x00, %edx - je L_AES_GCM_decrypt_calc_aad_done + je L_AES_GCM_decrypt_aesni_calc_aad_done xorl %ecx, %ecx cmpl $16, %edx - jl L_AES_GCM_decrypt_calc_aad_lt16 + jl L_AES_GCM_decrypt_aesni_calc_aad_lt16 andl $0xfffffff0, %edx -L_AES_GCM_decrypt_calc_aad_16_loop: +L_AES_GCM_decrypt_aesni_calc_aad_16_loop: movdqu (%r12,%rcx,1), %xmm8 pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 pxor %xmm8, %xmm6 @@ -2449,22 +2449,22 @@ L_AES_GCM_decrypt_calc_aad_16_loop: pxor %xmm2, %xmm6 addl $16, %ecx cmpl %edx, %ecx - jl L_AES_GCM_decrypt_calc_aad_16_loop + jl L_AES_GCM_decrypt_aesni_calc_aad_16_loop movl %r11d, %edx cmpl %edx, %ecx - je L_AES_GCM_decrypt_calc_aad_done -L_AES_GCM_decrypt_calc_aad_lt16: + je L_AES_GCM_decrypt_aesni_calc_aad_done +L_AES_GCM_decrypt_aesni_calc_aad_lt16: subq $16, %rsp pxor %xmm8, %xmm8 xorl %ebx, %ebx movdqu %xmm8, (%rsp) -L_AES_GCM_decrypt_calc_aad_loop: +L_AES_GCM_decrypt_aesni_calc_aad_loop: movzbl (%r12,%rcx,1), %r13d movb %r13b, (%rsp,%rbx,1) incl %ecx incl %ebx cmpl %edx, %ecx - jl L_AES_GCM_decrypt_calc_aad_loop + jl L_AES_GCM_decrypt_aesni_calc_aad_loop movdqu (%rsp), %xmm8 addq $16, %rsp pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 @@ -2523,7 +2523,7 @@ L_AES_GCM_decrypt_calc_aad_loop: pxor %xmm1, %xmm2 pxor %xmm7, %xmm2 pxor %xmm2, %xmm6 -L_AES_GCM_decrypt_calc_aad_done: +L_AES_GCM_decrypt_aesni_calc_aad_done: # Calculate counter and H pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 movdqa %xmm5, %xmm9 @@ -2541,7 +2541,7 @@ L_AES_GCM_decrypt_calc_aad_done: xorl %ebx, %ebx cmpl $0x80, %r9d movl %r9d, %r13d - jl L_AES_GCM_decrypt_done_128 + jl L_AES_GCM_decrypt_aesni_done_128 andl $0xffffff80, %r13d movdqa %xmm6, %xmm2 # H ^ 1 @@ -2840,7 +2840,7 @@ L_AES_GCM_decrypt_calc_aad_done: pxor %xmm8, %xmm14 pxor %xmm14, %xmm7 movdqu %xmm7, 112(%rsp) -L_AES_GCM_decrypt_ghash_128: +L_AES_GCM_decrypt_aesni_ghash_128: leaq (%rdi,%rbx,1), %rcx leaq (%rsi,%rbx,1), %rdx movdqu 128(%rsp), %xmm8 @@ -3108,7 +3108,7 @@ L_AES_GCM_decrypt_ghash_128: pxor %xmm3, %xmm2 cmpl $11, %r10d movdqa 160(%r15), %xmm7 - jl L_AES_GCM_decrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done aesenc %xmm7, %xmm8 aesenc %xmm7, %xmm9 aesenc %xmm7, %xmm10 @@ -3128,7 +3128,7 @@ L_AES_GCM_decrypt_ghash_128: aesenc %xmm7, %xmm15 cmpl $13, %r10d movdqa 192(%r15), %xmm7 - jl L_AES_GCM_decrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done aesenc %xmm7, %xmm8 aesenc %xmm7, %xmm9 aesenc %xmm7, %xmm10 @@ -3147,7 +3147,7 @@ L_AES_GCM_decrypt_ghash_128: aesenc %xmm7, %xmm14 aesenc %xmm7, %xmm15 movdqa 224(%r15), %xmm7 -L_AES_GCM_decrypt_aesenc_128_ghash_avx_done: +L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done: aesenclast %xmm7, %xmm8 aesenclast %xmm7, %xmm9 movdqu (%rcx), %xmm0 @@ -3182,18 +3182,18 @@ L_AES_GCM_decrypt_aesenc_128_ghash_avx_done: movdqu %xmm15, 112(%rdx) addl $0x80, %ebx cmpl %r13d, %ebx - jl L_AES_GCM_decrypt_ghash_128 + jl L_AES_GCM_decrypt_aesni_ghash_128 movdqa %xmm2, %xmm6 movdqu (%rsp), %xmm5 -L_AES_GCM_decrypt_done_128: +L_AES_GCM_decrypt_aesni_done_128: movl %r9d, %edx cmpl %edx, %ebx - jge L_AES_GCM_decrypt_done_dec + jge L_AES_GCM_decrypt_aesni_done_dec movl %r9d, %r13d andl $0xfffffff0, %r13d cmpl %r13d, %ebx - jge L_AES_GCM_decrypt_last_block_done -L_AES_GCM_decrypt_last_block_start: + jge L_AES_GCM_decrypt_aesni_last_block_done +L_AES_GCM_decrypt_aesni_last_block_start: leaq (%rdi,%rbx,1), %rcx leaq (%rsi,%rbx,1), %rdx movdqu (%rcx), %xmm1 @@ -3242,28 +3242,28 @@ L_AES_GCM_decrypt_last_block_start: pxor %xmm3, %xmm6 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_decrypt_aesenc_gfmul_last + jl L_AES_GCM_decrypt_aesni_aesenc_gfmul_last aesenc %xmm9, %xmm8 aesenc 176(%r15), %xmm8 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_decrypt_aesenc_gfmul_last + jl L_AES_GCM_decrypt_aesni_aesenc_gfmul_last aesenc %xmm9, %xmm8 aesenc 208(%r15), %xmm8 movdqa 224(%r15), %xmm9 -L_AES_GCM_decrypt_aesenc_gfmul_last: +L_AES_GCM_decrypt_aesni_aesenc_gfmul_last: aesenclast %xmm9, %xmm8 movdqu (%rcx), %xmm9 pxor %xmm9, %xmm8 movdqu %xmm8, (%rdx) addl $16, %ebx cmpl %r13d, %ebx - jl L_AES_GCM_decrypt_last_block_start -L_AES_GCM_decrypt_last_block_done: + jl L_AES_GCM_decrypt_aesni_last_block_start +L_AES_GCM_decrypt_aesni_last_block_done: movl %r9d, %ecx movl %ecx, %edx andl $15, %ecx - jz L_AES_GCM_decrypt_aesenc_last15_dec_avx_done + jz L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done movdqu 128(%rsp), %xmm4 pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 pxor (%r15), %xmm4 @@ -3278,23 +3278,23 @@ L_AES_GCM_decrypt_last_block_done: aesenc 144(%r15), %xmm4 cmpl $11, %r10d movdqa 160(%r15), %xmm9 - jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last aesenc %xmm9, %xmm4 aesenc 176(%r15), %xmm4 cmpl $13, %r10d movdqa 192(%r15), %xmm9 - jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last aesenc %xmm9, %xmm4 aesenc 208(%r15), %xmm4 movdqa 224(%r15), %xmm9 -L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last: +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last: aesenclast %xmm9, %xmm4 subq $32, %rsp xorl %ecx, %ecx movdqu %xmm4, (%rsp) pxor %xmm0, %xmm0 movdqu %xmm0, 16(%rsp) -L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop: movzbl (%rdi,%rbx,1), %r13d movb %r13b, 16(%rsp,%rcx,1) xorb (%rsp,%rcx,1), %r13b @@ -3302,7 +3302,7 @@ L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: incl %ebx incl %ecx cmpl %edx, %ebx - jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop + jl L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop movdqu 16(%rsp), %xmm4 addq $32, %rsp pshufb L_aes_gcm_bswap_mask(%rip), %xmm4 @@ -3347,8 +3347,8 @@ L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: pxor %xmm13, %xmm14 pxor %xmm8, %xmm14 pxor %xmm14, %xmm6 -L_AES_GCM_decrypt_aesenc_last15_dec_avx_done: -L_AES_GCM_decrypt_done_dec: +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_aesni_done_dec: movl %r9d, %edx movl %r11d, %ecx shlq $3, %rdx @@ -3400,24 +3400,24 @@ L_AES_GCM_decrypt_done_dec: movdqu 144(%rsp), %xmm0 pxor %xmm6, %xmm0 cmpl $16, %r14d - je L_AES_GCM_decrypt_cmp_tag_16 + je L_AES_GCM_decrypt_aesni_cmp_tag_16 subq $16, %rsp xorq %rcx, %rcx xorq %rbx, %rbx movdqu %xmm0, (%rsp) -L_AES_GCM_decrypt_cmp_tag_loop: +L_AES_GCM_decrypt_aesni_cmp_tag_loop: movzbl (%rsp,%rcx,1), %r13d xorb (%r8,%rcx,1), %r13b orb %r13b, %bl incl %ecx cmpl %r14d, %ecx - jne L_AES_GCM_decrypt_cmp_tag_loop + jne L_AES_GCM_decrypt_aesni_cmp_tag_loop cmpb $0x00, %bl sete %bl addq $16, %rsp xorq %rcx, %rcx - jmp L_AES_GCM_decrypt_cmp_tag_done -L_AES_GCM_decrypt_cmp_tag_16: + jmp L_AES_GCM_decrypt_aesni_cmp_tag_done +L_AES_GCM_decrypt_aesni_cmp_tag_16: movdqu (%r8), %xmm1 pcmpeqb %xmm1, %xmm0 pmovmskb %xmm0, %rdx @@ -3425,7 +3425,7 @@ L_AES_GCM_decrypt_cmp_tag_16: xorl %ebx, %ebx cmpl $0xffff, %edx sete %bl -L_AES_GCM_decrypt_cmp_tag_done: +L_AES_GCM_decrypt_aesni_cmp_tag_done: movl %ebx, (%rbp) addq $0xa8, %rsp popq %rbp @@ -3436,7 +3436,7 @@ L_AES_GCM_decrypt_cmp_tag_done: popq %r13 repz retq #ifndef __APPLE__ -.size AES_GCM_decrypt,.-AES_GCM_decrypt +.size AES_GCM_decrypt_aesni,.-AES_GCM_decrypt_aesni #endif /* __APPLE__ */ #ifdef WOLFSSL_AESGCM_STREAM #ifndef __APPLE__ diff --git a/wolfcrypt/src/aes_gcm_asm.asm b/wolfcrypt/src/aes_gcm_asm.asm index 348f2846d6..c0e3682fe3 100644 --- a/wolfcrypt/src/aes_gcm_asm.asm +++ b/wolfcrypt/src/aes_gcm_asm.asm @@ -96,7 +96,7 @@ L_aes_gcm_mod2_128 QWORD 1, 13979173243358019584 ptr_L_aes_gcm_mod2_128 QWORD L_aes_gcm_mod2_128 _DATA ENDS _text SEGMENT READONLY PARA -AES_GCM_encrypt PROC +AES_GCM_encrypt_aesni PROC push r13 push rdi push rsi @@ -130,7 +130,7 @@ AES_GCM_encrypt PROC pxor xmm6, xmm6 cmp ebx, 12 mov edx, ebx - jne L_AES_GCM_encrypt_iv_not_12 + jne L_AES_GCM_encrypt_aesni_iv_not_12 ; # Calculate values when IV is 12 bytes ; Set counter based on IV mov ecx, 16777216 @@ -170,7 +170,7 @@ AES_GCM_encrypt PROC aesenc xmm1, xmm7 cmp r10d, 11 movdqa xmm7, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_calc_iv_12_last + jl L_AES_GCM_encrypt_aesni_calc_iv_12_last aesenc xmm5, xmm7 aesenc xmm1, xmm7 movdqa xmm7, OWORD PTR [r15+176] @@ -178,20 +178,20 @@ AES_GCM_encrypt PROC aesenc xmm1, xmm7 cmp r10d, 13 movdqa xmm7, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_calc_iv_12_last + jl L_AES_GCM_encrypt_aesni_calc_iv_12_last aesenc xmm5, xmm7 aesenc xmm1, xmm7 movdqa xmm7, OWORD PTR [r15+208] aesenc xmm5, xmm7 aesenc xmm1, xmm7 movdqa xmm7, OWORD PTR [r15+224] -L_AES_GCM_encrypt_calc_iv_12_last: +L_AES_GCM_encrypt_aesni_calc_iv_12_last: aesenclast xmm5, xmm7 aesenclast xmm1, xmm7 pshufb xmm5, OWORD PTR L_aes_gcm_bswap_mask movdqu [rsp+144], xmm1 - jmp L_AES_GCM_encrypt_iv_done -L_AES_GCM_encrypt_iv_not_12: + jmp L_AES_GCM_encrypt_aesni_iv_done +L_AES_GCM_encrypt_aesni_iv_not_12: ; Calculate values when IV is not 12 bytes ; H = Encrypt X(=0) movdqa xmm5, OWORD PTR [r15] @@ -206,27 +206,27 @@ L_AES_GCM_encrypt_iv_not_12: aesenc xmm5, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last aesenc xmm5, xmm9 aesenc xmm5, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last aesenc xmm5, xmm9 aesenc xmm5, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last: aesenclast xmm5, xmm9 pshufb xmm5, OWORD PTR L_aes_gcm_bswap_mask ; Calc counter ; Initialization vector cmp edx, 0 mov rcx, 0 - je L_AES_GCM_encrypt_calc_iv_done + je L_AES_GCM_encrypt_aesni_calc_iv_done cmp edx, 16 - jl L_AES_GCM_encrypt_calc_iv_lt16 + jl L_AES_GCM_encrypt_aesni_calc_iv_lt16 and edx, 4294967280 -L_AES_GCM_encrypt_calc_iv_16_loop: +L_AES_GCM_encrypt_aesni_calc_iv_16_loop: movdqu xmm8, [rax+rcx] pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask pxor xmm4, xmm8 @@ -286,22 +286,22 @@ L_AES_GCM_encrypt_calc_iv_16_loop: pxor xmm4, xmm2 add ecx, 16 cmp ecx, edx - jl L_AES_GCM_encrypt_calc_iv_16_loop + jl L_AES_GCM_encrypt_aesni_calc_iv_16_loop mov edx, ebx cmp ecx, edx - je L_AES_GCM_encrypt_calc_iv_done -L_AES_GCM_encrypt_calc_iv_lt16: + je L_AES_GCM_encrypt_aesni_calc_iv_done +L_AES_GCM_encrypt_aesni_calc_iv_lt16: sub rsp, 16 pxor xmm8, xmm8 xor ebx, ebx movdqu [rsp], xmm8 -L_AES_GCM_encrypt_calc_iv_loop: +L_AES_GCM_encrypt_aesni_calc_iv_loop: movzx r13d, BYTE PTR [rax+rcx] mov BYTE PTR [rsp+rbx], r13b inc ecx inc ebx cmp ecx, edx - jl L_AES_GCM_encrypt_calc_iv_loop + jl L_AES_GCM_encrypt_aesni_calc_iv_loop movdqu xmm8, [rsp] add rsp, 16 pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask @@ -360,7 +360,7 @@ L_AES_GCM_encrypt_calc_iv_loop: pxor xmm2, xmm1 pxor xmm2, xmm7 pxor xmm4, xmm2 -L_AES_GCM_encrypt_calc_iv_done: +L_AES_GCM_encrypt_aesni_calc_iv_done: ; T = Encrypt counter pxor xmm0, xmm0 shl edx, 3 @@ -435,28 +435,28 @@ L_AES_GCM_encrypt_calc_iv_done: aesenc xmm8, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last aesenc xmm8, xmm9 aesenc xmm8, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last aesenc xmm8, xmm9 aesenc xmm8, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last: aesenclast xmm8, xmm9 movdqu [rsp+144], xmm8 -L_AES_GCM_encrypt_iv_done: +L_AES_GCM_encrypt_aesni_iv_done: ; Additional authentication data mov edx, r11d cmp edx, 0 - je L_AES_GCM_encrypt_calc_aad_done + je L_AES_GCM_encrypt_aesni_calc_aad_done xor ecx, ecx cmp edx, 16 - jl L_AES_GCM_encrypt_calc_aad_lt16 + jl L_AES_GCM_encrypt_aesni_calc_aad_lt16 and edx, 4294967280 -L_AES_GCM_encrypt_calc_aad_16_loop: +L_AES_GCM_encrypt_aesni_calc_aad_16_loop: movdqu xmm8, [r12+rcx] pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask pxor xmm6, xmm8 @@ -516,22 +516,22 @@ L_AES_GCM_encrypt_calc_aad_16_loop: pxor xmm6, xmm2 add ecx, 16 cmp ecx, edx - jl L_AES_GCM_encrypt_calc_aad_16_loop + jl L_AES_GCM_encrypt_aesni_calc_aad_16_loop mov edx, r11d cmp ecx, edx - je L_AES_GCM_encrypt_calc_aad_done -L_AES_GCM_encrypt_calc_aad_lt16: + je L_AES_GCM_encrypt_aesni_calc_aad_done +L_AES_GCM_encrypt_aesni_calc_aad_lt16: sub rsp, 16 pxor xmm8, xmm8 xor ebx, ebx movdqu [rsp], xmm8 -L_AES_GCM_encrypt_calc_aad_loop: +L_AES_GCM_encrypt_aesni_calc_aad_loop: movzx r13d, BYTE PTR [r12+rcx] mov BYTE PTR [rsp+rbx], r13b inc ecx inc ebx cmp ecx, edx - jl L_AES_GCM_encrypt_calc_aad_loop + jl L_AES_GCM_encrypt_aesni_calc_aad_loop movdqu xmm8, [rsp] add rsp, 16 pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask @@ -590,7 +590,7 @@ L_AES_GCM_encrypt_calc_aad_loop: pxor xmm2, xmm1 pxor xmm2, xmm7 pxor xmm6, xmm2 -L_AES_GCM_encrypt_calc_aad_done: +L_AES_GCM_encrypt_aesni_calc_aad_done: ; Calculate counter and H pshufb xmm4, OWORD PTR L_aes_gcm_bswap_epi64 movdqa xmm9, xmm5 @@ -608,7 +608,7 @@ L_AES_GCM_encrypt_calc_aad_done: xor rbx, rbx cmp r9d, 128 mov r13d, r9d - jl L_AES_GCM_encrypt_done_128 + jl L_AES_GCM_encrypt_aesni_done_128 and r13d, 4294967168 movdqa xmm2, xmm6 ; H ^ 1 @@ -1027,7 +1027,7 @@ L_AES_GCM_encrypt_calc_aad_done: aesenc xmm15, xmm7 cmp r10d, 11 movdqa xmm7, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_enc_done + jl L_AES_GCM_encrypt_aesni_enc_done aesenc xmm8, xmm7 aesenc xmm9, xmm7 aesenc xmm10, xmm7 @@ -1047,7 +1047,7 @@ L_AES_GCM_encrypt_calc_aad_done: aesenc xmm15, xmm7 cmp r10d, 13 movdqa xmm7, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_enc_done + jl L_AES_GCM_encrypt_aesni_enc_done aesenc xmm8, xmm7 aesenc xmm9, xmm7 aesenc xmm10, xmm7 @@ -1066,7 +1066,7 @@ L_AES_GCM_encrypt_calc_aad_done: aesenc xmm14, xmm7 aesenc xmm15, xmm7 movdqa xmm7, OWORD PTR [r15+224] -L_AES_GCM_encrypt_enc_done: +L_AES_GCM_encrypt_aesni_enc_done: aesenclast xmm8, xmm7 aesenclast xmm9, xmm7 movdqu xmm0, [rdi] @@ -1101,9 +1101,9 @@ L_AES_GCM_encrypt_enc_done: movdqu [rsi+112], xmm15 cmp r13d, 128 mov ebx, 128 - jle L_AES_GCM_encrypt_end_128 + jle L_AES_GCM_encrypt_aesni_end_128 ; More 128 bytes of input -L_AES_GCM_encrypt_ghash_128: +L_AES_GCM_encrypt_aesni_ghash_128: lea rcx, QWORD PTR [rdi+rbx] lea rdx, QWORD PTR [rsi+rbx] movdqu xmm8, [rsp+128] @@ -1371,7 +1371,7 @@ L_AES_GCM_encrypt_ghash_128: pxor xmm2, xmm3 cmp r10d, 11 movdqa xmm7, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done aesenc xmm8, xmm7 aesenc xmm9, xmm7 aesenc xmm10, xmm7 @@ -1391,7 +1391,7 @@ L_AES_GCM_encrypt_ghash_128: aesenc xmm15, xmm7 cmp r10d, 13 movdqa xmm7, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done aesenc xmm8, xmm7 aesenc xmm9, xmm7 aesenc xmm10, xmm7 @@ -1410,7 +1410,7 @@ L_AES_GCM_encrypt_ghash_128: aesenc xmm14, xmm7 aesenc xmm15, xmm7 movdqa xmm7, OWORD PTR [r15+224] -L_AES_GCM_encrypt_aesenc_128_ghash_avx_done: +L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done: aesenclast xmm8, xmm7 aesenclast xmm9, xmm7 movdqu xmm0, [rcx] @@ -1445,8 +1445,8 @@ L_AES_GCM_encrypt_aesenc_128_ghash_avx_done: movdqu [rdx+112], xmm15 add ebx, 128 cmp ebx, r13d - jl L_AES_GCM_encrypt_ghash_128 -L_AES_GCM_encrypt_end_128: + jl L_AES_GCM_encrypt_aesni_ghash_128 +L_AES_GCM_encrypt_aesni_end_128: movdqa xmm4, OWORD PTR L_aes_gcm_bswap_mask pshufb xmm8, xmm4 pshufb xmm9, xmm4 @@ -1633,14 +1633,14 @@ L_AES_GCM_encrypt_end_128: pxor xmm2, xmm4 pxor xmm6, xmm2 movdqu xmm5, [rsp] -L_AES_GCM_encrypt_done_128: +L_AES_GCM_encrypt_aesni_done_128: mov edx, r9d cmp ebx, edx - jge L_AES_GCM_encrypt_done_enc + jge L_AES_GCM_encrypt_aesni_done_enc mov r13d, r9d and r13d, 4294967280 cmp ebx, r13d - jge L_AES_GCM_encrypt_last_block_done + jge L_AES_GCM_encrypt_aesni_last_block_done lea rcx, QWORD PTR [rdi+rbx] lea rdx, QWORD PTR [rsi+rbx] movdqu xmm8, [rsp+128] @@ -1660,16 +1660,16 @@ L_AES_GCM_encrypt_done_128: aesenc xmm8, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last aesenc xmm8, xmm9 aesenc xmm8, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last aesenc xmm8, xmm9 aesenc xmm8, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last: aesenclast xmm8, xmm9 movdqu xmm9, [rcx] pxor xmm8, xmm9 @@ -1678,8 +1678,8 @@ L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last: pxor xmm6, xmm8 add ebx, 16 cmp ebx, r13d - jge L_AES_GCM_encrypt_last_block_ghash -L_AES_GCM_encrypt_last_block_start: + jge L_AES_GCM_encrypt_aesni_last_block_ghash +L_AES_GCM_encrypt_aesni_last_block_start: lea rcx, QWORD PTR [rdi+rbx] lea rdx, QWORD PTR [rsi+rbx] movdqu xmm8, [rsp+128] @@ -1724,16 +1724,16 @@ L_AES_GCM_encrypt_last_block_start: pxor xmm6, xmm3 cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_aesenc_gfmul_last + jl L_AES_GCM_encrypt_aesni_aesenc_gfmul_last aesenc xmm8, xmm9 aesenc xmm8, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_aesenc_gfmul_last + jl L_AES_GCM_encrypt_aesni_aesenc_gfmul_last aesenc xmm8, xmm9 aesenc xmm8, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_encrypt_aesenc_gfmul_last: +L_AES_GCM_encrypt_aesni_aesenc_gfmul_last: aesenclast xmm8, xmm9 movdqu xmm9, [rcx] pxor xmm8, xmm9 @@ -1742,8 +1742,8 @@ L_AES_GCM_encrypt_aesenc_gfmul_last: pxor xmm6, xmm8 add ebx, 16 cmp ebx, r13d - jl L_AES_GCM_encrypt_last_block_start -L_AES_GCM_encrypt_last_block_ghash: + jl L_AES_GCM_encrypt_aesni_last_block_start +L_AES_GCM_encrypt_aesni_last_block_ghash: pshufd xmm9, xmm5, 78 pshufd xmm10, xmm6, 78 movdqa xmm11, xmm6 @@ -1784,11 +1784,11 @@ L_AES_GCM_encrypt_last_block_ghash: pxor xmm14, xmm13 pxor xmm14, xmm8 pxor xmm6, xmm14 -L_AES_GCM_encrypt_last_block_done: +L_AES_GCM_encrypt_aesni_last_block_done: mov ecx, r9d mov edx, ecx and ecx, 15 - jz L_AES_GCM_encrypt_aesenc_last15_enc_avx_done + jz L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done movdqu xmm4, [rsp+128] pshufb xmm4, OWORD PTR L_aes_gcm_bswap_epi64 pxor xmm4, [r15] @@ -1803,21 +1803,21 @@ L_AES_GCM_encrypt_last_block_done: aesenc xmm4, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last aesenc xmm4, xmm9 aesenc xmm4, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last aesenc xmm4, xmm9 aesenc xmm4, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last: +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last: aesenclast xmm4, xmm9 sub rsp, 16 xor ecx, ecx movdqu [rsp], xmm4 -L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop: +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop: movzx r13d, BYTE PTR [rdi+rbx] xor r13b, BYTE PTR [rsp+rcx] mov BYTE PTR [rsi+rbx], r13b @@ -1825,16 +1825,16 @@ L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop: inc ebx inc ecx cmp ebx, edx - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop xor r13, r13 cmp ecx, 16 - je L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc -L_AES_GCM_encrypt_aesenc_last15_enc_avx_byte_loop: + je L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop: mov BYTE PTR [rsp+rcx], r13b inc ecx cmp ecx, 16 - jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_byte_loop -L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc: + jl L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc: movdqu xmm4, [rsp] add rsp, 16 pshufb xmm4, OWORD PTR L_aes_gcm_bswap_mask @@ -1879,8 +1879,8 @@ L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc: pxor xmm14, xmm13 pxor xmm14, xmm8 pxor xmm6, xmm14 -L_AES_GCM_encrypt_aesenc_last15_enc_avx_done: -L_AES_GCM_encrypt_done_enc: +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_aesni_done_enc: mov edx, r9d mov ecx, r11d shl rdx, 3 @@ -1932,19 +1932,19 @@ L_AES_GCM_encrypt_done_enc: movdqu xmm0, [rsp+144] pxor xmm0, xmm6 cmp r14d, 16 - je L_AES_GCM_encrypt_store_tag_16 + je L_AES_GCM_encrypt_aesni_store_tag_16 xor rcx, rcx movdqu [rsp], xmm0 -L_AES_GCM_encrypt_store_tag_loop: +L_AES_GCM_encrypt_aesni_store_tag_loop: movzx r13d, BYTE PTR [rsp+rcx] mov BYTE PTR [r8+rcx], r13b inc ecx cmp ecx, r14d - jne L_AES_GCM_encrypt_store_tag_loop - jmp L_AES_GCM_encrypt_store_tag_done -L_AES_GCM_encrypt_store_tag_16: + jne L_AES_GCM_encrypt_aesni_store_tag_loop + jmp L_AES_GCM_encrypt_aesni_store_tag_done +L_AES_GCM_encrypt_aesni_store_tag_16: movdqu [r8], xmm0 -L_AES_GCM_encrypt_store_tag_done: +L_AES_GCM_encrypt_aesni_store_tag_done: movdqu xmm6, [rsp+160] movdqu xmm7, [rsp+176] movdqu xmm8, [rsp+192] @@ -1964,10 +1964,10 @@ L_AES_GCM_encrypt_store_tag_done: pop rdi pop r13 ret -AES_GCM_encrypt ENDP +AES_GCM_encrypt_aesni ENDP _text ENDS _text SEGMENT READONLY PARA -AES_GCM_decrypt PROC +AES_GCM_decrypt_aesni PROC push r13 push rdi push rsi @@ -2003,7 +2003,7 @@ AES_GCM_decrypt PROC pxor xmm6, xmm6 cmp ebx, 12 mov edx, ebx - jne L_AES_GCM_decrypt_iv_not_12 + jne L_AES_GCM_decrypt_aesni_iv_not_12 ; # Calculate values when IV is 12 bytes ; Set counter based on IV mov ecx, 16777216 @@ -2043,7 +2043,7 @@ AES_GCM_decrypt PROC aesenc xmm1, xmm7 cmp r10d, 11 movdqa xmm7, OWORD PTR [r15+160] - jl L_AES_GCM_decrypt_calc_iv_12_last + jl L_AES_GCM_decrypt_aesni_calc_iv_12_last aesenc xmm5, xmm7 aesenc xmm1, xmm7 movdqa xmm7, OWORD PTR [r15+176] @@ -2051,20 +2051,20 @@ AES_GCM_decrypt PROC aesenc xmm1, xmm7 cmp r10d, 13 movdqa xmm7, OWORD PTR [r15+192] - jl L_AES_GCM_decrypt_calc_iv_12_last + jl L_AES_GCM_decrypt_aesni_calc_iv_12_last aesenc xmm5, xmm7 aesenc xmm1, xmm7 movdqa xmm7, OWORD PTR [r15+208] aesenc xmm5, xmm7 aesenc xmm1, xmm7 movdqa xmm7, OWORD PTR [r15+224] -L_AES_GCM_decrypt_calc_iv_12_last: +L_AES_GCM_decrypt_aesni_calc_iv_12_last: aesenclast xmm5, xmm7 aesenclast xmm1, xmm7 pshufb xmm5, OWORD PTR L_aes_gcm_bswap_mask movdqu [rsp+144], xmm1 - jmp L_AES_GCM_decrypt_iv_done -L_AES_GCM_decrypt_iv_not_12: + jmp L_AES_GCM_decrypt_aesni_iv_done +L_AES_GCM_decrypt_aesni_iv_not_12: ; Calculate values when IV is not 12 bytes ; H = Encrypt X(=0) movdqa xmm5, OWORD PTR [r15] @@ -2079,27 +2079,27 @@ L_AES_GCM_decrypt_iv_not_12: aesenc xmm5, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last aesenc xmm5, xmm9 aesenc xmm5, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last aesenc xmm5, xmm9 aesenc xmm5, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last: +L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last: aesenclast xmm5, xmm9 pshufb xmm5, OWORD PTR L_aes_gcm_bswap_mask ; Calc counter ; Initialization vector cmp edx, 0 mov rcx, 0 - je L_AES_GCM_decrypt_calc_iv_done + je L_AES_GCM_decrypt_aesni_calc_iv_done cmp edx, 16 - jl L_AES_GCM_decrypt_calc_iv_lt16 + jl L_AES_GCM_decrypt_aesni_calc_iv_lt16 and edx, 4294967280 -L_AES_GCM_decrypt_calc_iv_16_loop: +L_AES_GCM_decrypt_aesni_calc_iv_16_loop: movdqu xmm8, [rax+rcx] pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask pxor xmm4, xmm8 @@ -2159,22 +2159,22 @@ L_AES_GCM_decrypt_calc_iv_16_loop: pxor xmm4, xmm2 add ecx, 16 cmp ecx, edx - jl L_AES_GCM_decrypt_calc_iv_16_loop + jl L_AES_GCM_decrypt_aesni_calc_iv_16_loop mov edx, ebx cmp ecx, edx - je L_AES_GCM_decrypt_calc_iv_done -L_AES_GCM_decrypt_calc_iv_lt16: + je L_AES_GCM_decrypt_aesni_calc_iv_done +L_AES_GCM_decrypt_aesni_calc_iv_lt16: sub rsp, 16 pxor xmm8, xmm8 xor ebx, ebx movdqu [rsp], xmm8 -L_AES_GCM_decrypt_calc_iv_loop: +L_AES_GCM_decrypt_aesni_calc_iv_loop: movzx r13d, BYTE PTR [rax+rcx] mov BYTE PTR [rsp+rbx], r13b inc ecx inc ebx cmp ecx, edx - jl L_AES_GCM_decrypt_calc_iv_loop + jl L_AES_GCM_decrypt_aesni_calc_iv_loop movdqu xmm8, [rsp] add rsp, 16 pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask @@ -2233,7 +2233,7 @@ L_AES_GCM_decrypt_calc_iv_loop: pxor xmm2, xmm1 pxor xmm2, xmm7 pxor xmm4, xmm2 -L_AES_GCM_decrypt_calc_iv_done: +L_AES_GCM_decrypt_aesni_calc_iv_done: ; T = Encrypt counter pxor xmm0, xmm0 shl edx, 3 @@ -2308,28 +2308,28 @@ L_AES_GCM_decrypt_calc_iv_done: aesenc xmm8, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last aesenc xmm8, xmm9 aesenc xmm8, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last aesenc xmm8, xmm9 aesenc xmm8, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last: +L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last: aesenclast xmm8, xmm9 movdqu [rsp+144], xmm8 -L_AES_GCM_decrypt_iv_done: +L_AES_GCM_decrypt_aesni_iv_done: ; Additional authentication data mov edx, r11d cmp edx, 0 - je L_AES_GCM_decrypt_calc_aad_done + je L_AES_GCM_decrypt_aesni_calc_aad_done xor ecx, ecx cmp edx, 16 - jl L_AES_GCM_decrypt_calc_aad_lt16 + jl L_AES_GCM_decrypt_aesni_calc_aad_lt16 and edx, 4294967280 -L_AES_GCM_decrypt_calc_aad_16_loop: +L_AES_GCM_decrypt_aesni_calc_aad_16_loop: movdqu xmm8, [r12+rcx] pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask pxor xmm6, xmm8 @@ -2389,22 +2389,22 @@ L_AES_GCM_decrypt_calc_aad_16_loop: pxor xmm6, xmm2 add ecx, 16 cmp ecx, edx - jl L_AES_GCM_decrypt_calc_aad_16_loop + jl L_AES_GCM_decrypt_aesni_calc_aad_16_loop mov edx, r11d cmp ecx, edx - je L_AES_GCM_decrypt_calc_aad_done -L_AES_GCM_decrypt_calc_aad_lt16: + je L_AES_GCM_decrypt_aesni_calc_aad_done +L_AES_GCM_decrypt_aesni_calc_aad_lt16: sub rsp, 16 pxor xmm8, xmm8 xor ebx, ebx movdqu [rsp], xmm8 -L_AES_GCM_decrypt_calc_aad_loop: +L_AES_GCM_decrypt_aesni_calc_aad_loop: movzx r13d, BYTE PTR [r12+rcx] mov BYTE PTR [rsp+rbx], r13b inc ecx inc ebx cmp ecx, edx - jl L_AES_GCM_decrypt_calc_aad_loop + jl L_AES_GCM_decrypt_aesni_calc_aad_loop movdqu xmm8, [rsp] add rsp, 16 pshufb xmm8, OWORD PTR L_aes_gcm_bswap_mask @@ -2463,7 +2463,7 @@ L_AES_GCM_decrypt_calc_aad_loop: pxor xmm2, xmm1 pxor xmm2, xmm7 pxor xmm6, xmm2 -L_AES_GCM_decrypt_calc_aad_done: +L_AES_GCM_decrypt_aesni_calc_aad_done: ; Calculate counter and H pshufb xmm4, OWORD PTR L_aes_gcm_bswap_epi64 movdqa xmm9, xmm5 @@ -2481,7 +2481,7 @@ L_AES_GCM_decrypt_calc_aad_done: xor ebx, ebx cmp r9d, 128 mov r13d, r9d - jl L_AES_GCM_decrypt_done_128 + jl L_AES_GCM_decrypt_aesni_done_128 and r13d, 4294967168 movdqa xmm2, xmm6 ; H ^ 1 @@ -2780,7 +2780,7 @@ L_AES_GCM_decrypt_calc_aad_done: pxor xmm14, xmm8 pxor xmm7, xmm14 movdqu [rsp+112], xmm7 -L_AES_GCM_decrypt_ghash_128: +L_AES_GCM_decrypt_aesni_ghash_128: lea rcx, QWORD PTR [rdi+rbx] lea rdx, QWORD PTR [rsi+rbx] movdqu xmm8, [rsp+128] @@ -3048,7 +3048,7 @@ L_AES_GCM_decrypt_ghash_128: pxor xmm2, xmm3 cmp r10d, 11 movdqa xmm7, OWORD PTR [r15+160] - jl L_AES_GCM_decrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done aesenc xmm8, xmm7 aesenc xmm9, xmm7 aesenc xmm10, xmm7 @@ -3068,7 +3068,7 @@ L_AES_GCM_decrypt_ghash_128: aesenc xmm15, xmm7 cmp r10d, 13 movdqa xmm7, OWORD PTR [r15+192] - jl L_AES_GCM_decrypt_aesenc_128_ghash_avx_done + jl L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done aesenc xmm8, xmm7 aesenc xmm9, xmm7 aesenc xmm10, xmm7 @@ -3087,7 +3087,7 @@ L_AES_GCM_decrypt_ghash_128: aesenc xmm14, xmm7 aesenc xmm15, xmm7 movdqa xmm7, OWORD PTR [r15+224] -L_AES_GCM_decrypt_aesenc_128_ghash_avx_done: +L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done: aesenclast xmm8, xmm7 aesenclast xmm9, xmm7 movdqu xmm0, [rcx] @@ -3122,18 +3122,18 @@ L_AES_GCM_decrypt_aesenc_128_ghash_avx_done: movdqu [rdx+112], xmm15 add ebx, 128 cmp ebx, r13d - jl L_AES_GCM_decrypt_ghash_128 + jl L_AES_GCM_decrypt_aesni_ghash_128 movdqa xmm6, xmm2 movdqu xmm5, [rsp] -L_AES_GCM_decrypt_done_128: +L_AES_GCM_decrypt_aesni_done_128: mov edx, r9d cmp ebx, edx - jge L_AES_GCM_decrypt_done_dec + jge L_AES_GCM_decrypt_aesni_done_dec mov r13d, r9d and r13d, 4294967280 cmp ebx, r13d - jge L_AES_GCM_decrypt_last_block_done -L_AES_GCM_decrypt_last_block_start: + jge L_AES_GCM_decrypt_aesni_last_block_done +L_AES_GCM_decrypt_aesni_last_block_start: lea rcx, QWORD PTR [rdi+rbx] lea rdx, QWORD PTR [rsi+rbx] movdqu xmm1, [rcx] @@ -3182,28 +3182,28 @@ L_AES_GCM_decrypt_last_block_start: pxor xmm6, xmm3 cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_decrypt_aesenc_gfmul_last + jl L_AES_GCM_decrypt_aesni_aesenc_gfmul_last aesenc xmm8, xmm9 aesenc xmm8, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_decrypt_aesenc_gfmul_last + jl L_AES_GCM_decrypt_aesni_aesenc_gfmul_last aesenc xmm8, xmm9 aesenc xmm8, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_decrypt_aesenc_gfmul_last: +L_AES_GCM_decrypt_aesni_aesenc_gfmul_last: aesenclast xmm8, xmm9 movdqu xmm9, [rcx] pxor xmm8, xmm9 movdqu [rdx], xmm8 add ebx, 16 cmp ebx, r13d - jl L_AES_GCM_decrypt_last_block_start -L_AES_GCM_decrypt_last_block_done: + jl L_AES_GCM_decrypt_aesni_last_block_start +L_AES_GCM_decrypt_aesni_last_block_done: mov ecx, r9d mov edx, ecx and ecx, 15 - jz L_AES_GCM_decrypt_aesenc_last15_dec_avx_done + jz L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done movdqu xmm4, [rsp+128] pshufb xmm4, OWORD PTR L_aes_gcm_bswap_epi64 pxor xmm4, [r15] @@ -3218,23 +3218,23 @@ L_AES_GCM_decrypt_last_block_done: aesenc xmm4, [r15+144] cmp r10d, 11 movdqa xmm9, OWORD PTR [r15+160] - jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last aesenc xmm4, xmm9 aesenc xmm4, [r15+176] cmp r10d, 13 movdqa xmm9, OWORD PTR [r15+192] - jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last + jl L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last aesenc xmm4, xmm9 aesenc xmm4, [r15+208] movdqa xmm9, OWORD PTR [r15+224] -L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last: +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last: aesenclast xmm4, xmm9 sub rsp, 32 xor ecx, ecx movdqu [rsp], xmm4 pxor xmm0, xmm0 movdqu [rsp+16], xmm0 -L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop: movzx r13d, BYTE PTR [rdi+rbx] mov BYTE PTR [rsp+rcx+16], r13b xor r13b, BYTE PTR [rsp+rcx] @@ -3242,7 +3242,7 @@ L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: inc ebx inc ecx cmp ebx, edx - jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop + jl L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop movdqu xmm4, [rsp+16] add rsp, 32 pshufb xmm4, OWORD PTR L_aes_gcm_bswap_mask @@ -3287,8 +3287,8 @@ L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: pxor xmm14, xmm13 pxor xmm14, xmm8 pxor xmm6, xmm14 -L_AES_GCM_decrypt_aesenc_last15_dec_avx_done: -L_AES_GCM_decrypt_done_dec: +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_aesni_done_dec: mov edx, r9d mov ecx, r11d shl rdx, 3 @@ -3340,24 +3340,24 @@ L_AES_GCM_decrypt_done_dec: movdqu xmm0, [rsp+144] pxor xmm0, xmm6 cmp r14d, 16 - je L_AES_GCM_decrypt_cmp_tag_16 + je L_AES_GCM_decrypt_aesni_cmp_tag_16 sub rsp, 16 xor rcx, rcx xor rbx, rbx movdqu [rsp], xmm0 -L_AES_GCM_decrypt_cmp_tag_loop: +L_AES_GCM_decrypt_aesni_cmp_tag_loop: movzx r13d, BYTE PTR [rsp+rcx] xor r13b, BYTE PTR [r8+rcx] or bl, r13b inc ecx cmp ecx, r14d - jne L_AES_GCM_decrypt_cmp_tag_loop + jne L_AES_GCM_decrypt_aesni_cmp_tag_loop cmp rbx, 0 sete bl add rsp, 16 xor rcx, rcx - jmp L_AES_GCM_decrypt_cmp_tag_done -L_AES_GCM_decrypt_cmp_tag_16: + jmp L_AES_GCM_decrypt_aesni_cmp_tag_done +L_AES_GCM_decrypt_aesni_cmp_tag_16: movdqu xmm1, [r8] pcmpeqb xmm0, xmm1 pmovmskb rdx, xmm0 @@ -3365,7 +3365,7 @@ L_AES_GCM_decrypt_cmp_tag_16: xor ebx, ebx cmp edx, 65535 sete bl -L_AES_GCM_decrypt_cmp_tag_done: +L_AES_GCM_decrypt_aesni_cmp_tag_done: mov DWORD PTR [rbp], ebx movdqu xmm6, [rsp+168] movdqu xmm7, [rsp+184] @@ -3387,7 +3387,7 @@ L_AES_GCM_decrypt_cmp_tag_done: pop rdi pop r13 ret -AES_GCM_decrypt ENDP +AES_GCM_decrypt_aesni ENDP _text ENDS _text SEGMENT READONLY PARA AES_GCM_init_aesni PROC diff --git a/wolfcrypt/src/aes_xts_asm.S b/wolfcrypt/src/aes_xts_asm.S index bba34a2e80..fedead84f8 100644 --- a/wolfcrypt/src/aes_xts_asm.S +++ b/wolfcrypt/src/aes_xts_asm.S @@ -56,15 +56,15 @@ L_aes_xts_gc_xts: .long 0x87,0x1,0x1,0x1 #ifndef __APPLE__ .text -.globl AES_XTS_encrypt -.type AES_XTS_encrypt,@function +.globl AES_XTS_encrypt_aesni +.type AES_XTS_encrypt_aesni,@function .align 16 -AES_XTS_encrypt: +AES_XTS_encrypt_aesni: #else .section __TEXT,__text -.globl _AES_XTS_encrypt +.globl _AES_XTS_encrypt_aesni .p2align 4 -_AES_XTS_encrypt: +_AES_XTS_encrypt_aesni: #endif /* __APPLE__ */ pushq %r12 pushq %r13 @@ -96,25 +96,25 @@ _AES_XTS_encrypt: aesenc %xmm5, %xmm0 cmpl $11, %r10d movdqu 160(%r9), %xmm5 - jl L_AES_XTS_encrypt_tweak_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last aesenc %xmm5, %xmm0 movdqu 176(%r9), %xmm6 aesenc %xmm6, %xmm0 cmpl $13, %r10d movdqu 192(%r9), %xmm5 - jl L_AES_XTS_encrypt_tweak_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last aesenc %xmm5, %xmm0 movdqu 208(%r9), %xmm6 aesenc %xmm6, %xmm0 movdqu 224(%r9), %xmm5 -L_AES_XTS_encrypt_tweak_aes_enc_block_last: +L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last: aesenclast %xmm5, %xmm0 xorl %r13d, %r13d cmpl $0x40, %eax movl %eax, %r11d - jl L_AES_XTS_encrypt_done_64 + jl L_AES_XTS_encrypt_aesni_done_64 andl $0xffffffc0, %r11d -L_AES_XTS_encrypt_enc_64: +L_AES_XTS_encrypt_aesni_enc_64: # 64 bytes of input # aes_enc_64 leaq (%rdi,%r13,1), %rcx @@ -201,7 +201,7 @@ L_AES_XTS_encrypt_enc_64: aesenc %xmm4, %xmm11 cmpl $11, %r10d movdqu 160(%r8), %xmm4 - jl L_AES_XTS_encrypt_aes_enc_64_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last aesenc %xmm4, %xmm8 aesenc %xmm4, %xmm9 aesenc %xmm4, %xmm10 @@ -213,7 +213,7 @@ L_AES_XTS_encrypt_enc_64: aesenc %xmm4, %xmm11 cmpl $13, %r10d movdqu 192(%r8), %xmm4 - jl L_AES_XTS_encrypt_aes_enc_64_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last aesenc %xmm4, %xmm8 aesenc %xmm4, %xmm9 aesenc %xmm4, %xmm10 @@ -224,7 +224,7 @@ L_AES_XTS_encrypt_enc_64: aesenc %xmm4, %xmm10 aesenc %xmm4, %xmm11 movdqu 224(%r8), %xmm4 -L_AES_XTS_encrypt_aes_enc_64_aes_enc_block_last: +L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last: aesenclast %xmm4, %xmm8 aesenclast %xmm4, %xmm9 aesenclast %xmm4, %xmm10 @@ -246,18 +246,18 @@ L_AES_XTS_encrypt_aes_enc_64_aes_enc_block_last: pxor %xmm4, %xmm0 addl $0x40, %r13d cmpl %r11d, %r13d - jl L_AES_XTS_encrypt_enc_64 -L_AES_XTS_encrypt_done_64: + jl L_AES_XTS_encrypt_aesni_enc_64 +L_AES_XTS_encrypt_aesni_done_64: cmpl %eax, %r13d movl %eax, %r11d - je L_AES_XTS_encrypt_done_enc + je L_AES_XTS_encrypt_aesni_done_enc subl %r13d, %r11d cmpl $16, %r11d movl %eax, %r11d - jl L_AES_XTS_encrypt_last_15 + jl L_AES_XTS_encrypt_aesni_last_15 andl $0xfffffff0, %r11d # 16 bytes of input -L_AES_XTS_encrypt_enc_16: +L_AES_XTS_encrypt_aesni_enc_16: leaq (%rdi,%r13,1), %rcx movdqu (%rcx), %xmm8 pxor %xmm0, %xmm8 @@ -283,18 +283,18 @@ L_AES_XTS_encrypt_enc_16: aesenc %xmm5, %xmm8 cmpl $11, %r10d movdqu 160(%r8), %xmm5 - jl L_AES_XTS_encrypt_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_aes_enc_block_last aesenc %xmm5, %xmm8 movdqu 176(%r8), %xmm6 aesenc %xmm6, %xmm8 cmpl $13, %r10d movdqu 192(%r8), %xmm5 - jl L_AES_XTS_encrypt_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_aes_enc_block_last aesenc %xmm5, %xmm8 movdqu 208(%r8), %xmm6 aesenc %xmm6, %xmm8 movdqu 224(%r8), %xmm5 -L_AES_XTS_encrypt_aes_enc_block_last: +L_AES_XTS_encrypt_aesni_aes_enc_block_last: aesenclast %xmm5, %xmm8 pxor %xmm0, %xmm8 leaq (%rsi,%r13,1), %rcx @@ -307,17 +307,17 @@ L_AES_XTS_encrypt_aes_enc_block_last: pxor %xmm4, %xmm0 addl $16, %r13d cmpl %r11d, %r13d - jl L_AES_XTS_encrypt_enc_16 + jl L_AES_XTS_encrypt_aesni_enc_16 cmpl %eax, %r13d - je L_AES_XTS_encrypt_done_enc -L_AES_XTS_encrypt_last_15: + je L_AES_XTS_encrypt_aesni_done_enc +L_AES_XTS_encrypt_aesni_last_15: subq $16, %r13 leaq (%rsi,%r13,1), %rcx movdqu (%rcx), %xmm8 addq $16, %r13 movdqu %xmm8, (%rsp) xorq %rdx, %rdx -L_AES_XTS_encrypt_last_15_byte_loop: +L_AES_XTS_encrypt_aesni_last_15_byte_loop: movb (%rsp,%rdx,1), %r11b movb (%rdi,%r13,1), %cl movb %r11b, (%rsi,%r13,1) @@ -325,7 +325,7 @@ L_AES_XTS_encrypt_last_15_byte_loop: incl %r13d incl %edx cmpl %eax, %r13d - jl L_AES_XTS_encrypt_last_15_byte_loop + jl L_AES_XTS_encrypt_aesni_last_15_byte_loop subq %rdx, %r13 movdqu (%rsp), %xmm8 subq $16, %r13 @@ -352,41 +352,41 @@ L_AES_XTS_encrypt_last_15_byte_loop: aesenc %xmm5, %xmm8 cmpl $11, %r10d movdqu 160(%r8), %xmm5 - jl L_AES_XTS_encrypt_last_15_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last aesenc %xmm5, %xmm8 movdqu 176(%r8), %xmm6 aesenc %xmm6, %xmm8 cmpl $13, %r10d movdqu 192(%r8), %xmm5 - jl L_AES_XTS_encrypt_last_15_aes_enc_block_last + jl L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last aesenc %xmm5, %xmm8 movdqu 208(%r8), %xmm6 aesenc %xmm6, %xmm8 movdqu 224(%r8), %xmm5 -L_AES_XTS_encrypt_last_15_aes_enc_block_last: +L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last: aesenclast %xmm5, %xmm8 pxor %xmm0, %xmm8 leaq (%rsi,%r13,1), %rcx movdqu %xmm8, (%rcx) -L_AES_XTS_encrypt_done_enc: +L_AES_XTS_encrypt_aesni_done_enc: addq $0x40, %rsp popq %r13 popq %r12 repz retq #ifndef __APPLE__ -.size AES_XTS_encrypt,.-AES_XTS_encrypt +.size AES_XTS_encrypt_aesni,.-AES_XTS_encrypt_aesni #endif /* __APPLE__ */ #ifndef __APPLE__ .text -.globl AES_XTS_decrypt -.type AES_XTS_decrypt,@function +.globl AES_XTS_decrypt_aesni +.type AES_XTS_decrypt_aesni,@function .align 16 -AES_XTS_decrypt: +AES_XTS_decrypt_aesni: #else .section __TEXT,__text -.globl _AES_XTS_decrypt +.globl _AES_XTS_decrypt_aesni .p2align 4 -_AES_XTS_decrypt: +_AES_XTS_decrypt_aesni: #endif /* __APPLE__ */ pushq %r12 pushq %r13 @@ -418,32 +418,32 @@ _AES_XTS_decrypt: aesenc %xmm5, %xmm0 cmpl $11, %r10d movdqu 160(%r9), %xmm5 - jl L_AES_XTS_decrypt_tweak_aes_enc_block_last + jl L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last aesenc %xmm5, %xmm0 movdqu 176(%r9), %xmm6 aesenc %xmm6, %xmm0 cmpl $13, %r10d movdqu 192(%r9), %xmm5 - jl L_AES_XTS_decrypt_tweak_aes_enc_block_last + jl L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last aesenc %xmm5, %xmm0 movdqu 208(%r9), %xmm6 aesenc %xmm6, %xmm0 movdqu 224(%r9), %xmm5 -L_AES_XTS_decrypt_tweak_aes_enc_block_last: +L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last: aesenclast %xmm5, %xmm0 xorl %r13d, %r13d movl %eax, %r11d andl $0xfffffff0, %r11d cmpl %eax, %r11d - je L_AES_XTS_decrypt_mul16_64 + je L_AES_XTS_decrypt_aesni_mul16_64 subl $16, %r11d cmpl $16, %r11d - jl L_AES_XTS_decrypt_last_31_start -L_AES_XTS_decrypt_mul16_64: + jl L_AES_XTS_decrypt_aesni_last_31_start +L_AES_XTS_decrypt_aesni_mul16_64: cmpl $0x40, %r11d - jl L_AES_XTS_decrypt_done_64 + jl L_AES_XTS_decrypt_aesni_done_64 andl $0xffffffc0, %r11d -L_AES_XTS_decrypt_dec_64: +L_AES_XTS_decrypt_aesni_dec_64: # 64 bytes of input # aes_dec_64 leaq (%rdi,%r13,1), %rcx @@ -530,7 +530,7 @@ L_AES_XTS_decrypt_dec_64: aesdec %xmm4, %xmm11 cmpl $11, %r10d movdqu 160(%r8), %xmm4 - jl L_AES_XTS_decrypt_aes_dec_64_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last aesdec %xmm4, %xmm8 aesdec %xmm4, %xmm9 aesdec %xmm4, %xmm10 @@ -542,7 +542,7 @@ L_AES_XTS_decrypt_dec_64: aesdec %xmm4, %xmm11 cmpl $13, %r10d movdqu 192(%r8), %xmm4 - jl L_AES_XTS_decrypt_aes_dec_64_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last aesdec %xmm4, %xmm8 aesdec %xmm4, %xmm9 aesdec %xmm4, %xmm10 @@ -553,7 +553,7 @@ L_AES_XTS_decrypt_dec_64: aesdec %xmm4, %xmm10 aesdec %xmm4, %xmm11 movdqu 224(%r8), %xmm4 -L_AES_XTS_decrypt_aes_dec_64_aes_dec_block_last: +L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last: aesdeclast %xmm4, %xmm8 aesdeclast %xmm4, %xmm9 aesdeclast %xmm4, %xmm10 @@ -575,21 +575,21 @@ L_AES_XTS_decrypt_aes_dec_64_aes_dec_block_last: pxor %xmm4, %xmm0 addl $0x40, %r13d cmpl %r11d, %r13d - jl L_AES_XTS_decrypt_dec_64 -L_AES_XTS_decrypt_done_64: + jl L_AES_XTS_decrypt_aesni_dec_64 +L_AES_XTS_decrypt_aesni_done_64: cmpl %eax, %r13d movl %eax, %r11d - je L_AES_XTS_decrypt_done_dec + je L_AES_XTS_decrypt_aesni_done_dec andl $0xfffffff0, %r11d cmpl %eax, %r11d - je L_AES_XTS_decrypt_mul16 + je L_AES_XTS_decrypt_aesni_mul16 subl $16, %r11d subl %r13d, %r11d cmpl $16, %r11d - jl L_AES_XTS_decrypt_last_31_start + jl L_AES_XTS_decrypt_aesni_last_31_start addl %r13d, %r11d -L_AES_XTS_decrypt_mul16: -L_AES_XTS_decrypt_dec_16: +L_AES_XTS_decrypt_aesni_mul16: +L_AES_XTS_decrypt_aesni_dec_16: # 16 bytes of input leaq (%rdi,%r13,1), %rcx movdqu (%rcx), %xmm8 @@ -616,18 +616,18 @@ L_AES_XTS_decrypt_dec_16: aesdec %xmm5, %xmm8 cmpl $11, %r10d movdqu 160(%r8), %xmm5 - jl L_AES_XTS_decrypt_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_aes_dec_block_last aesdec %xmm5, %xmm8 movdqu 176(%r8), %xmm6 aesdec %xmm6, %xmm8 cmpl $13, %r10d movdqu 192(%r8), %xmm5 - jl L_AES_XTS_decrypt_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_aes_dec_block_last aesdec %xmm5, %xmm8 movdqu 208(%r8), %xmm6 aesdec %xmm6, %xmm8 movdqu 224(%r8), %xmm5 -L_AES_XTS_decrypt_aes_dec_block_last: +L_AES_XTS_decrypt_aesni_aes_dec_block_last: aesdeclast %xmm5, %xmm8 pxor %xmm0, %xmm8 leaq (%rsi,%r13,1), %rcx @@ -640,10 +640,10 @@ L_AES_XTS_decrypt_aes_dec_block_last: pxor %xmm4, %xmm0 addl $16, %r13d cmpl %r11d, %r13d - jl L_AES_XTS_decrypt_dec_16 + jl L_AES_XTS_decrypt_aesni_dec_16 cmpl %eax, %r13d - je L_AES_XTS_decrypt_done_dec -L_AES_XTS_decrypt_last_31_start: + je L_AES_XTS_decrypt_aesni_done_dec +L_AES_XTS_decrypt_aesni_last_31_start: movdqa %xmm0, %xmm4 movdqa %xmm0, %xmm7 psrad $31, %xmm4 @@ -676,24 +676,24 @@ L_AES_XTS_decrypt_last_31_start: aesdec %xmm5, %xmm8 cmpl $11, %r10d movdqu 160(%r8), %xmm5 - jl L_AES_XTS_decrypt_last_31_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last aesdec %xmm5, %xmm8 movdqu 176(%r8), %xmm6 aesdec %xmm6, %xmm8 cmpl $13, %r10d movdqu 192(%r8), %xmm5 - jl L_AES_XTS_decrypt_last_31_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last aesdec %xmm5, %xmm8 movdqu 208(%r8), %xmm6 aesdec %xmm6, %xmm8 movdqu 224(%r8), %xmm5 -L_AES_XTS_decrypt_last_31_aes_dec_block_last: +L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last: aesdeclast %xmm5, %xmm8 pxor %xmm7, %xmm8 movdqu %xmm8, (%rsp) addq $16, %r13 xorq %rdx, %rdx -L_AES_XTS_decrypt_last_31_byte_loop: +L_AES_XTS_decrypt_aesni_last_31_byte_loop: movb (%rsp,%rdx,1), %r11b movb (%rdi,%r13,1), %cl movb %r11b, (%rsi,%r13,1) @@ -701,7 +701,7 @@ L_AES_XTS_decrypt_last_31_byte_loop: incl %r13d incl %edx cmpl %eax, %r13d - jl L_AES_XTS_decrypt_last_31_byte_loop + jl L_AES_XTS_decrypt_aesni_last_31_byte_loop subq %rdx, %r13 movdqu (%rsp), %xmm8 pxor %xmm0, %xmm8 @@ -727,30 +727,30 @@ L_AES_XTS_decrypt_last_31_byte_loop: aesdec %xmm5, %xmm8 cmpl $11, %r10d movdqu 160(%r8), %xmm5 - jl L_AES_XTS_decrypt_last_31_2_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last aesdec %xmm5, %xmm8 movdqu 176(%r8), %xmm6 aesdec %xmm6, %xmm8 cmpl $13, %r10d movdqu 192(%r8), %xmm5 - jl L_AES_XTS_decrypt_last_31_2_aes_dec_block_last + jl L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last aesdec %xmm5, %xmm8 movdqu 208(%r8), %xmm6 aesdec %xmm6, %xmm8 movdqu 224(%r8), %xmm5 -L_AES_XTS_decrypt_last_31_2_aes_dec_block_last: +L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last: aesdeclast %xmm5, %xmm8 pxor %xmm0, %xmm8 subq $16, %r13 leaq (%rsi,%r13,1), %rcx movdqu %xmm8, (%rcx) -L_AES_XTS_decrypt_done_dec: +L_AES_XTS_decrypt_aesni_done_dec: addq $16, %rsp popq %r13 popq %r12 repz retq #ifndef __APPLE__ -.size AES_XTS_decrypt,.-AES_XTS_decrypt +.size AES_XTS_decrypt_aesni,.-AES_XTS_decrypt_aesni #endif /* __APPLE__ */ #ifdef HAVE_INTEL_AVX1 #ifndef __APPLE__ diff --git a/wolfcrypt/src/asn.c b/wolfcrypt/src/asn.c index abee0c94e6..021ec4af22 100644 --- a/wolfcrypt/src/asn.c +++ b/wolfcrypt/src/asn.c @@ -30979,6 +30979,9 @@ int wc_SetSubjectKeyId(Cert *cert, const char* file) #ifdef HAVE_ECC wc_ecc_free(eckey); XFREE(eckey, cert->heap, DYNAMIC_TYPE_ECC); +#endif +#if defined(NO_RSA) && !defined(HAVE_ECC) + (void)idx; #endif return ret; } @@ -32191,7 +32194,7 @@ int DecodeECC_DSA_Sig_Ex(const byte* sig, word32 sigLen, mp_int* r, mp_int* s, #ifdef WOLFSSL_ASN_TEMPLATE -#ifdef WOLFSSL_CUSTOM_CURVES +#if defined(HAVE_ECC) && defined(WOLFSSL_CUSTOM_CURVES) /* Convert data to hex string. * * Big-endian byte array is converted to big-endian hexadecimal string. diff --git a/wolfcrypt/src/evp.c b/wolfcrypt/src/evp.c index f92fb04929..dcdb575883 100644 --- a/wolfcrypt/src/evp.c +++ b/wolfcrypt/src/evp.c @@ -4672,7 +4672,7 @@ int wolfSSL_EVP_read_pw_string(char* buf, int bufSz, const char* banner, int v) } #endif /* WOLFSSL_APACHE_HTTPD */ -#if !defined(NO_PWDBASED) && !defined(NO_SHA) +#if !defined(NO_PWDBASED) && !defined(NO_SHA) && !defined(NO_HMAC) int wolfSSL_PKCS5_PBKDF2_HMAC_SHA1(const char *pass, int passlen, const unsigned char *salt, int saltlen, int iter, @@ -4698,7 +4698,7 @@ int wolfSSL_PKCS5_PBKDF2_HMAC_SHA1(const char *pass, int passlen, } #endif /* !NO_PWDBASED !NO_SHA*/ -#if !defined(NO_PWDBASED) +#if !defined(NO_PWDBASED) && !defined(NO_HMAC) int wolfSSL_PKCS5_PBKDF2_HMAC(const char *pass, int passlen, const unsigned char *salt, int saltlen, int iter, diff --git a/wolfcrypt/src/logging.c b/wolfcrypt/src/logging.c index eacc6b09d9..87727a36bb 100644 --- a/wolfcrypt/src/logging.c +++ b/wolfcrypt/src/logging.c @@ -136,13 +136,6 @@ static struct log mynewt_log; #endif /* DEBUG_WOLFSSL */ -#ifdef DEBUG_VECTOR_REGISTER_ACCESS -THREAD_LS_T int wc_svr_count = 0; -THREAD_LS_T const char *wc_svr_last_file = NULL; -THREAD_LS_T int wc_svr_last_line = -1; -#endif - - /* allow this to be set to NULL, so logs can be redirected to default output */ int wolfSSL_SetLoggingCb(wolfSSL_Logging_cb f) { @@ -1546,4 +1539,3 @@ void WOLFSSL_ERROR_MSG(const char* msg) } #endif /* DEBUG_WOLFSSL || WOLFSSL_NGINX || WOLFSSL_HAPROXY */ - diff --git a/wolfcrypt/src/memory.c b/wolfcrypt/src/memory.c index ae3412cd09..3c265c0cf3 100644 --- a/wolfcrypt/src/memory.c +++ b/wolfcrypt/src/memory.c @@ -31,7 +31,7 @@ #define WOLFSSL_NEED_LINUX_CURRENT #endif -#include +#include /* Possible memory options: @@ -1231,7 +1231,6 @@ void* wolfSSL_Realloc(void *ptr, size_t size, void* heap, int type) /* Example for user io pool, shared build may need definitions in lib proper */ -#include #include #ifndef HAVE_THREAD_LS @@ -1439,6 +1438,42 @@ void __attribute__((no_instrument_function)) } #endif +#ifdef DEBUG_VECTOR_REGISTER_ACCESS +THREAD_LS_T int wc_svr_count = 0; +THREAD_LS_T const char *wc_svr_last_file = NULL; +THREAD_LS_T int wc_svr_last_line = -1; +THREAD_LS_T int wc_debug_vector_registers_retval = + WC_DEBUG_VECTOR_REGISTERS_RETVAL_INITVAL; + +#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING + +WOLFSSL_LOCAL int SAVE_VECTOR_REGISTERS2_fuzzer(void) { + static THREAD_LS_T struct drand48_data wc_svr_fuzzing_state; + static THREAD_LS_T int wc_svr_fuzzing_seeded = 0; + long result; + + if (wc_debug_vector_registers_retval) + return wc_debug_vector_registers_retval; + + if (wc_svr_fuzzing_seeded == 0) { + long seed = WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED; + char *seed_envstr = getenv("WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED"); + if (seed_envstr) + seed = strtol(seed_envstr, NULL, 0); + (void)srand48_r(seed, &wc_svr_fuzzing_state); + wc_svr_fuzzing_seeded = 1; + } + (void)lrand48_r(&wc_svr_fuzzing_state, &result); + if (result & 1) + return IO_FAILED_E; + else + return 0; +} + +#endif + +#endif + #ifdef WOLFSSL_LINUXKM #include "../../linuxkm/linuxkm_memory.c" #endif diff --git a/wolfcrypt/src/pwdbased.c b/wolfcrypt/src/pwdbased.c index e6ede27311..8be0c64e82 100644 --- a/wolfcrypt/src/pwdbased.c +++ b/wolfcrypt/src/pwdbased.c @@ -559,6 +559,10 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen, #endif /* HAVE_PKCS12 */ #ifdef HAVE_SCRYPT +#ifdef NO_HMAC + #error scrypt requires HMAC +#endif + /* Rotate the 32-bit value a by b bits to the left. * * a 32-bit value. diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 8de3e5f7c9..c2956f2a49 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -574,7 +574,9 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_evpSig_test(void); WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pbkdf1_test(void); WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pkcs12_test(void); +#if defined(HAVE_PBKDF2) && !defined(NO_SHA256) && !defined(NO_HMAC) WOLFSSL_TEST_SUBROUTINE wc_test_ret_t pbkdf2_test(void); +#endif WOLFSSL_TEST_SUBROUTINE wc_test_ret_t scrypt_test(void); #ifdef HAVE_ECC WOLFSSL_TEST_SUBROUTINE wc_test_ret_t ecc_test(void); @@ -9466,6 +9468,19 @@ static wc_test_ret_t aes_xts_128_test(void) if (XMEMCMP(c2, buf, sizeof(c2))) ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesXtsEncrypt(aes, buf, p2, sizeof(p2), i2, sizeof(i2)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(c2, buf, sizeof(c2))) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + XMEMSET(buf, 0, sizeof(buf)); wc_AesXtsFree(aes); @@ -9482,6 +9497,19 @@ static wc_test_ret_t aes_xts_128_test(void) if (XMEMCMP(c1, buf, AES_BLOCK_SIZE)) ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesXtsEncrypt(aes, buf, p1, sizeof(p1), i1, sizeof(i1)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(c1, buf, AES_BLOCK_SIZE)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + /* partial block encryption test */ XMEMSET(cipher, 0, sizeof(cipher)); ret = wc_AesXtsEncrypt(aes, cipher, pp, sizeof(pp), i1, sizeof(i1)); @@ -9492,6 +9520,21 @@ static wc_test_ret_t aes_xts_128_test(void) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); if (XMEMCMP(cp2, cipher, sizeof(cp2))) ERROR_OUT(WC_TEST_RET_ENC_NC, out); + +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + XMEMSET(cipher, 0, sizeof(cipher)); + ret = wc_AesXtsEncrypt(aes, cipher, pp, sizeof(pp), i1, sizeof(i1)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(cp2, cipher, sizeof(cp2))) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + wc_AesXtsFree(aes); /* partial block decrypt test */ @@ -9509,6 +9552,20 @@ static wc_test_ret_t aes_xts_128_test(void) if (XMEMCMP(pp, buf, sizeof(pp))) ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + XMEMSET(buf, 0, sizeof(buf)); + ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(pp, buf, sizeof(pp))) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + /* NIST decrypt test vector */ XMEMSET(buf, 0, sizeof(buf)); ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1)); @@ -9520,6 +9577,20 @@ static wc_test_ret_t aes_xts_128_test(void) if (XMEMCMP(p1, buf, AES_BLOCK_SIZE)) ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + XMEMSET(buf, 0, sizeof(buf)); + ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(p1, buf, AES_BLOCK_SIZE)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + /* fail case with decrypting using wrong key */ XMEMSET(buf, 0, sizeof(buf)); ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2)); @@ -10368,17 +10439,83 @@ static wc_test_ret_t aesecb_test(void) if (XMEMCMP(cipher, niCipher, AES_BLOCK_SIZE) != 0) ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + XMEMSET(cipher, 0, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesSetKey(enc, niKey, sizeof(niKey), cipher, AES_ENCRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesEcbEncrypt(enc, cipher, niPlain, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (XMEMCMP(cipher, niCipher, AES_BLOCK_SIZE) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + + XMEMSET(cipher, 0, AES_BLOCK_SIZE); + ret = wc_AesEcbEncrypt(enc, cipher, niPlain, AES_BLOCK_SIZE); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (XMEMCMP(cipher, niCipher, AES_BLOCK_SIZE) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + + XMEMSET(cipher, 0, AES_BLOCK_SIZE); + ret = wc_AesSetKey(enc, niKey, sizeof(niKey), cipher, AES_ENCRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesEcbEncrypt(enc, cipher, niPlain, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (XMEMCMP(cipher, niCipher, AES_BLOCK_SIZE) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + XMEMSET(plain, 0, AES_BLOCK_SIZE); ret = wc_AesSetKey(dec, niKey, sizeof(niKey), plain, AES_DECRYPTION); if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); if (wc_AesEcbDecrypt(dec, plain, niCipher, AES_BLOCK_SIZE) != 0) ERROR_OUT(WC_TEST_RET_ENC_NC, out); - wc_AesEcbDecrypt(dec, plain, niCipher, AES_BLOCK_SIZE); if (XMEMCMP(plain, niPlain, AES_BLOCK_SIZE) != 0) ERROR_OUT(WC_TEST_RET_ENC_NC, out); - } +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + XMEMSET(plain, 0, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesSetKey(dec, niKey, sizeof(niKey), plain, AES_DECRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesEcbDecrypt(dec, plain, niCipher, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (XMEMCMP(plain, niPlain, AES_BLOCK_SIZE) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + + XMEMSET(plain, 0, AES_BLOCK_SIZE); + ret = wc_AesEcbDecrypt(dec, plain, niCipher, AES_BLOCK_SIZE); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (XMEMCMP(plain, niPlain, AES_BLOCK_SIZE) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + + XMEMSET(plain, 0, AES_BLOCK_SIZE); + ret = wc_AesSetKey(dec, niKey, sizeof(niKey), plain, AES_DECRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesEcbDecrypt(dec, plain, niCipher, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (XMEMCMP(plain, niPlain, AES_BLOCK_SIZE) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + } out: #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC) @@ -11009,6 +11146,86 @@ static wc_test_ret_t aesctr_test(Aes* enc, Aes* dec, byte* cipher, byte* plain) #endif } +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + for (i = 0; i < AES_CTR_TEST_LEN; i++) { + if (testVec[i].key != NULL) { + ret = wc_AesSetKeyDirect(enc, testVec[i].key, testVec[i].keySz, + testVec[i].iv, AES_ENCRYPTION); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + /* Ctr only uses encrypt, even on key setup */ + ret = wc_AesSetKeyDirect(dec, testVec[i].key, testVec[i].keySz, + testVec[i].iv, AES_ENCRYPTION); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + } + + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesCtrEncrypt(enc, cipher, testVec[i].plain, testVec[i].len); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesCtrEncrypt(dec, plain, cipher, testVec[i].len); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + + if (XMEMCMP(plain, ctrPlain, testVec[i].len)) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } +#if !(FIPS_VERSION_EQ(2,0) && defined(WOLFSSL_ARMASM)) + if (XMEMCMP(cipher, testVec[i].cipher, testVec[i].len)) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } +#endif + } + + for (i = 0; i < AES_CTR_TEST_LEN; i++) { + if (testVec[i].key != NULL) { + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesSetKeyDirect(enc, testVec[i].key, testVec[i].keySz, + testVec[i].iv, AES_ENCRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + /* Ctr only uses encrypt, even on key setup */ + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesSetKeyDirect(dec, testVec[i].key, testVec[i].keySz, + testVec[i].iv, AES_ENCRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + } + + ret = wc_AesCtrEncrypt(enc, cipher, testVec[i].plain, testVec[i].len); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + ret = wc_AesCtrEncrypt(dec, plain, cipher, testVec[i].len); + if (ret != 0) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } + + if (XMEMCMP(plain, ctrPlain, testVec[i].len)) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } +#if !(FIPS_VERSION_EQ(2,0) && defined(WOLFSSL_ARMASM)) + if (XMEMCMP(cipher, testVec[i].cipher, testVec[i].len)) { + ERROR_OUT(WC_TEST_RET_ENC_I(i), out); + } +#endif + } + +#endif /* DEBUG_VECTOR_REGISTER_ACCESS && WC_AES_C_DYNAMIC_FALLBACK */ + + out: return ret; } @@ -11262,6 +11479,57 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes_test(void) break; } +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + /* Iterate from one AES_BLOCK_SIZE of bigMsg through the whole + * message by AES_BLOCK_SIZE for each size of AES key. */ + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + for (keySz = 16; keySz <= 32; keySz += 8) { + for (msgSz = AES_BLOCK_SIZE; + msgSz <= sizeof(bigMsg); + msgSz += AES_BLOCK_SIZE) { + + XMEMSET(bigCipher, 0, sizeof(bigMsg)); + XMEMSET(bigPlain, 0, sizeof(bigMsg)); + ret = wc_AesSetKey(enc, bigKey, keySz, iv, AES_ENCRYPTION); + if (ret != 0) { + ret = WC_TEST_RET_ENC_EC(ret); + break; + } + ret = wc_AesSetKey(dec, bigKey, keySz, iv, AES_DECRYPTION); + if (ret != 0) { + ret = WC_TEST_RET_ENC_EC(ret); + break; + } + + ret = wc_AesCbcEncrypt(enc, bigCipher, bigMsg, msgSz); + #if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &enc->asyncDev, WC_ASYNC_FLAG_NONE); + #endif + if (ret != 0) { + ret = WC_TEST_RET_ENC_EC(ret); + break; + } + + ret = wc_AesCbcDecrypt(dec, bigPlain, bigCipher, msgSz); + #if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &dec->asyncDev, WC_ASYNC_FLAG_NONE); + #endif + if (ret != 0) { + ret = WC_TEST_RET_ENC_EC(ret); + break; + } + + if (XMEMCMP(bigPlain, bigMsg, msgSz)) { + ret = WC_TEST_RET_ENC_NC; + break; + } + } + if (ret != 0) + break; + } + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); +#endif /* DEBUG_VECTOR_REGISTER_ACCESS && WC_AES_C_DYNAMIC_FALLBACK */ + #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC) XFREE(bigCipher, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); XFREE(bigPlain, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); @@ -11591,6 +11859,9 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes192_test(void) #endif if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(cipher, verify, (int) sizeof(cipher))) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + #ifdef HAVE_AES_DECRYPT XMEMSET(plain, 0, AES_BLOCK_SIZE); ret = wc_AesCbcDecrypt(dec, plain, cipher, (int) sizeof(cipher)); @@ -11604,9 +11875,6 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes192_test(void) } #endif - if (XMEMCMP(cipher, verify, (int) sizeof(cipher))) - ERROR_OUT(WC_TEST_RET_ENC_NC, out); - wc_AesFree(enc); #ifdef HAVE_AES_DECRYPT wc_AesFree(dec); @@ -11732,6 +12000,93 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t aes256_test(void) wc_AesFree(enc); #ifdef HAVE_AES_DECRYPT wc_AesFree(dec); +#endif + +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + ret = wc_AesSetKey(enc, key, keySz, iv, AES_ENCRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); +#ifdef HAVE_AES_DECRYPT + ret = wc_AesSetKey(dec, key, keySz, iv, AES_DECRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); +#endif + + XMEMSET(cipher, 0, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesCbcEncrypt(enc, cipher, msg, (int) sizeof(msg)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &enc->asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); +#ifdef HAVE_AES_DECRYPT + XMEMSET(plain, 0, AES_BLOCK_SIZE); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesCbcDecrypt(dec, plain, cipher, (int) sizeof(cipher)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &dec->asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(plain, msg, (int) sizeof(plain))) { + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } +#endif +#ifndef HAVE_RENESAS_SYNC + if (XMEMCMP(cipher, verify, (int) sizeof(cipher))) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + + wc_AesFree(enc); +#ifdef HAVE_AES_DECRYPT + wc_AesFree(dec); +#endif + + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesSetKey(enc, key, keySz, iv, AES_ENCRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); +#ifdef HAVE_AES_DECRYPT + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesSetKey(dec, key, keySz, iv, AES_DECRYPTION); + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); +#endif + + XMEMSET(cipher, 0, AES_BLOCK_SIZE); + ret = wc_AesCbcEncrypt(enc, cipher, msg, (int) sizeof(msg)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &enc->asyncDev, WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); +#ifdef HAVE_AES_DECRYPT + XMEMSET(plain, 0, AES_BLOCK_SIZE); + ret = wc_AesCbcDecrypt(dec, plain, cipher, (int) sizeof(cipher)); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &dec->asyncDev, WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(plain, msg, (int) sizeof(plain))) { + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } +#endif +#ifndef HAVE_RENESAS_SYNC + if (XMEMCMP(cipher, verify, (int) sizeof(cipher))) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif + + wc_AesFree(enc); +#ifdef HAVE_AES_DECRYPT + wc_AesFree(dec); +#endif + #endif out: @@ -11800,7 +12155,6 @@ static wc_test_ret_t aesgcm_default_test_helper(byte* key, int keySz, byte* iv, /* AES-GCM encrypt and decrypt both use AES encrypt internally */ ret = wc_AesGcmEncrypt(enc, resultC, plain, plainSz, iv, ivSz, resultT, tagSz, aad, aadSz); - #if defined(WOLFSSL_ASYNC_CRYPT) ret = wc_AsyncWait(ret, &enc->asyncDev, WC_ASYNC_FLAG_NONE); #endif @@ -11813,6 +12167,23 @@ static wc_test_ret_t aesgcm_default_test_helper(byte* key, int keySz, byte* iv, if (XMEMCMP(tag, resultT, tagSz)) ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesGcmEncrypt(enc, resultC, plain, plainSz, iv, ivSz, + resultT, tagSz, aad, aadSz); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &enc->asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (cipher != NULL) { + if (XMEMCMP(cipher, resultC, cipherSz)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } + if (XMEMCMP(tag, resultT, tagSz)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif #ifdef HAVE_AES_DECRYPT ret = wc_AesGcmSetKey(dec, key, keySz); @@ -11831,6 +12202,22 @@ static wc_test_ret_t aesgcm_default_test_helper(byte* key, int keySz, byte* iv, ERROR_OUT(WC_TEST_RET_ENC_NC, out); } +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_AES_C_DYNAMIC_FALLBACK) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(SYSLIB_FAILED_E); + ret = wc_AesGcmDecrypt(dec, resultP, resultC, cipherSz, + iv, ivSz, resultT, tagSz, aad, aadSz); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &dec->asyncDev, WC_ASYNC_FLAG_NONE); +#endif + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (plain != NULL) { + if (XMEMCMP(plain, resultP, plainSz)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } +#endif + #endif /* HAVE_AES_DECRYPT */ ret = 0; @@ -20202,6 +20589,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) word32 bytes; word32 idx = 0; WC_RNG rng; + int rng_inited = 0; wc_Sha sha; byte hash[WC_SHA_DIGEST_SIZE]; byte signature[40]; @@ -20209,6 +20597,11 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) byte* der = 0; #endif #define DSA_TEST_TMP_SIZE 1024 + int key_inited = 0; + int derIn_inited = 0; +#ifdef WOLFSSL_KEY_GEN + int genKey_inited = 0; +#endif #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC) byte *tmp = (byte *)XMALLOC(DSA_TEST_TMP_SIZE, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); DsaKey *key = (DsaKey *)XMALLOC(sizeof *key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); @@ -20216,6 +20609,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) DsaKey *derIn = (DsaKey *)XMALLOC(sizeof *derIn, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); DsaKey *genKey = (DsaKey *)XMALLOC(sizeof *genKey, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); #endif + if ((tmp == NULL) || (key == NULL) #ifdef WOLFSSL_KEY_GEN @@ -20264,6 +20658,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) ret = wc_InitDsaKey(key); if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + key_inited = 1; ret = wc_DsaPrivateKeyDecode(tmp, &idx, key, bytes); if (ret != 0) @@ -20276,6 +20671,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) #endif if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + rng_inited = 1; ret = wc_DsaSign(hash, signature, key, &rng); if (ret != 0) @@ -20287,8 +20683,6 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) if (answer != 1) ERROR_OUT(WC_TEST_RET_ENC_NC, out); - wc_FreeDsaKey(key); - #ifdef WOLFSSL_KEY_GEN { int derSz = 0; @@ -20296,56 +20690,39 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) ret = wc_InitDsaKey(genKey); if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + genKey_inited = 1; ret = wc_MakeDsaParameters(&rng, 1024, genKey); - if (ret != 0) { - wc_FreeDsaKey(genKey); + if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); - } ret = wc_MakeDsaKey(&rng, genKey); - if (ret != 0) { - wc_FreeDsaKey(genKey); + if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); - } der = (byte*)XMALLOC(FOURK_BUF, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - if (der == NULL) { - wc_FreeDsaKey(genKey); + if (der == NULL) ERROR_OUT(WC_TEST_RET_ENC_NC, out); - } derSz = wc_DsaKeyToDer(genKey, der, FOURK_BUF); - if (derSz < 0) { - XFREE(der, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (derSz < 0) ERROR_OUT(WC_TEST_RET_ENC_EC(derSz), out); - } ret = SaveDerAndPem(der, derSz, keyDerFile, keyPemFile, DSA_PRIVATEKEY_TYPE); - if (ret != 0) { - XFREE(der, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - wc_FreeDsaKey(genKey); + if (ret != 0) goto out; - } ret = wc_InitDsaKey(derIn); - if (ret != 0) { - XFREE(der, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - wc_FreeDsaKey(genKey); + if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); - } + derIn_inited = 1; idx = 0; ret = wc_DsaPrivateKeyDecode(der, &idx, derIn, derSz); - if (ret != 0) { - XFREE(der, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - wc_FreeDsaKey(derIn); - wc_FreeDsaKey(genKey); + if (ret != 0) ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); } - - } #endif /* WOLFSSL_KEY_GEN */ out: @@ -20368,15 +20745,20 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC) if (tmp) XFREE(tmp, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - if (key) + if (key) { + if (key_inited) + wc_FreeDsaKey(key); XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + } #ifdef WOLFSSL_KEY_GEN if (derIn) { - wc_FreeDsaKey(derIn); + if (derIn_inited) + wc_FreeDsaKey(derIn); XFREE(derIn, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); } if (genKey) { - wc_FreeDsaKey(genKey); + if (genKey_inited) + wc_FreeDsaKey(genKey); XFREE(genKey, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); } #endif @@ -20384,13 +20766,18 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t dsa_test(void) #else /* !WOLFSSL_SMALL_STACK || WOLFSSL_NO_MALLOC */ #ifdef WOLFSSL_KEY_GEN - wc_FreeDsaKey(derIn); - wc_FreeDsaKey(genKey); + if (key_inited) + wc_FreeDsaKey(key); + if (derIn_inited) + wc_FreeDsaKey(derIn); + if (genKey_inited) + wc_FreeDsaKey(genKey); #endif #endif - wc_FreeRng(&rng); + if (rng_inited) + wc_FreeRng(&rng); return ret; } @@ -41352,7 +41739,7 @@ static wc_test_ret_t pkcs7authenveloped_run_vectors(byte* rsaCert, word32 rsaCer #endif #if !defined(NO_AES) && defined(WOLFSSL_AES_256) && defined(HAVE_ECC) && \ - defined(WOLFSSL_SHA512) + defined(WOLFSSL_SHA512) && defined(HAVE_AESGCM) WOLFSSL_SMALL_STACK_STATIC const byte optionalUkm[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 }; @@ -41455,7 +41842,12 @@ static wc_test_ret_t pkcs7authenveloped_run_vectors(byte* rsaCert, word32 rsaCer NULL, 0, 0, NULL, 0, NULL, 0, 0, 0, 0, 0, 0, 0, 0, "pkcs7authEnvelopedDataAES256GCM_IANDS.der"); #endif - #endif /* NO_AES */ + #else /* NO_AES || !HAVE_AESGCM */ + (void)rsaCert; + (void)rsaCertSz; + (void)rsaPrivKey; + (void)rsaPrivKeySz; + #endif /* NO_AES || !HAVE_AESGCM */ #endif /* key agreement key encryption technique*/ @@ -48972,7 +49364,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cryptocb_test(void) #endif #endif #ifndef NO_PWDBASED - #if defined(HAVE_PBKDF2) && !defined(NO_SHA256) + #if defined(HAVE_PBKDF2) && !defined(NO_SHA256) && !defined(NO_HMAC) if (ret == 0) ret = pbkdf2_test(); #endif diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index ab4016cdcd..4c5e358650 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -243,13 +243,15 @@ enum { #endif struct Aes { - /* AESNI needs key first, rounds 2nd, not sure why yet */ ALIGN16 word32 key[60]; #ifdef WC_AES_BITSLICED /* Extra key schedule space required for bit-slicing technique. */ ALIGN16 bs_word bs_key[15 * AES_BLOCK_SIZE * BS_WORD_SIZE]; #endif word32 rounds; +#ifdef WC_AES_C_DYNAMIC_FALLBACK + word32 key_C_fallback[60]; +#endif int keylen; ALIGN16 word32 reg[AES_BLOCK_SIZE / sizeof(word32)]; /* for CBC mode */ diff --git a/wolfssl/wolfcrypt/cryptocb.h b/wolfssl/wolfcrypt/cryptocb.h index 0f468c4e6a..cf38444f0d 100644 --- a/wolfssl/wolfcrypt/cryptocb.h +++ b/wolfssl/wolfcrypt/cryptocb.h @@ -89,7 +89,6 @@ typedef struct wc_CryptoInfo { #if HAVE_ANONYMOUS_INLINE_AGGREGATES union { #endif -#if !defined(NO_RSA) || defined(HAVE_ECC) struct { int type; /* enum wc_PkType */ #if HAVE_ANONYMOUS_INLINE_AGGREGATES @@ -206,7 +205,6 @@ typedef struct wc_CryptoInfo { }; #endif } pk; -#endif /* !NO_RSA || HAVE_ECC */ #if !defined(NO_AES) || !defined(NO_DES3) struct { int type; /* enum wc_CipherType */ diff --git a/wolfssl/wolfcrypt/error-crypt.h b/wolfssl/wolfcrypt/error-crypt.h index f689a40bb0..d29ac13b81 100644 --- a/wolfssl/wolfcrypt/error-crypt.h +++ b/wolfssl/wolfcrypt/error-crypt.h @@ -238,19 +238,19 @@ enum { BAD_LENGTH_E = -279, /* Value of length parameter is invalid. */ ECDSA_KAT_FIPS_E = -280, /* ECDSA KAT failure */ RSA_PAT_FIPS_E = -281, /* RSA Pairwise failure */ - KDF_TLS12_KAT_FIPS_E = -282, /* TLS12 KDF KAT failure */ - KDF_TLS13_KAT_FIPS_E = -283, /* TLS13 KDF KAT failure */ + KDF_TLS12_KAT_FIPS_E = -282, /* TLS12 KDF KAT failure */ + KDF_TLS13_KAT_FIPS_E = -283, /* TLS13 KDF KAT failure */ KDF_SSH_KAT_FIPS_E = -284, /* SSH KDF KAT failure */ DHE_PCT_E = -285, /* DHE Pairwise Consistency Test failure */ ECC_PCT_E = -286, /* ECDHE Pairwise Consistency Test failure */ FIPS_PRIVATE_KEY_LOCKED_E = -287, /* Cannot export private key. */ PROTOCOLCB_UNAVAILABLE = -288, /* Protocol callback unavailable */ - AES_SIV_AUTH_E = -289, /* AES-SIV authentication failed */ - NO_VALID_DEVID = -290, /* no valid device ID */ + AES_SIV_AUTH_E = -289, /* AES-SIV authentication failed */ + NO_VALID_DEVID = -290, /* no valid device ID */ - IO_FAILED_E = -291, /* Input/output failure */ - SYSLIB_FAILED_E = -292, /* System/library call failed */ - USE_HW_PSK = -293, /* Callback return to indicate HW has PSK */ + IO_FAILED_E = -291, /* Input/output failure */ + SYSLIB_FAILED_E = -292, /* System/library call failed */ + USE_HW_PSK = -293, /* Callback return to indicate HW has PSK */ ENTROPY_RT_E = -294, /* Entropy Repetition Test failed */ ENTROPY_APT_E = -295, /* Entropy Adaptive Proportion Test failed */ diff --git a/wolfssl/wolfcrypt/memory.h b/wolfssl/wolfcrypt/memory.h index 15978b8376..bd72ccb664 100644 --- a/wolfssl/wolfcrypt/memory.h +++ b/wolfssl/wolfcrypt/memory.h @@ -251,9 +251,173 @@ WOLFSSL_LOCAL void wc_MemZero_Add(const char* name, const void* addr, WOLFSSL_LOCAL void wc_MemZero_Check(void* addr, size_t len); #endif +#ifdef DEBUG_VECTOR_REGISTER_ACCESS + WOLFSSL_API extern THREAD_LS_T int wc_svr_count; + WOLFSSL_API extern THREAD_LS_T const char *wc_svr_last_file; + WOLFSSL_API extern THREAD_LS_T int wc_svr_last_line; + + #ifdef DEBUG_VECTOR_REGISTERS_ABORT_ON_FAIL + #define DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE abort(); + #elif defined(DEBUG_VECTOR_REGISTERS_EXIT_ON_FAIL) + #define DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE exit(1); + #else + #define DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE + #endif + + #define SAVE_VECTOR_REGISTERS(fail_clause) { \ + int _svr_ret = wc_debug_vector_registers_retval; \ + if (_svr_ret != 0) { fail_clause } \ + ++wc_svr_count; \ + if (wc_svr_count > 5) { \ + fprintf(stderr, \ + ("%s @ L%d : incr : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + } \ + wc_svr_last_file = __FILE__; \ + wc_svr_last_line = __LINE__; \ + } + + WOLFSSL_API extern THREAD_LS_T int wc_debug_vector_registers_retval; + +#ifndef WC_DEBUG_VECTOR_REGISTERS_RETVAL_INITVAL +#define WC_DEBUG_VECTOR_REGISTERS_RETVAL_INITVAL 0 +#endif +#define WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(x) do { \ + if (((x) != 0) && (wc_svr_count > 0)) { \ + fprintf(stderr, \ + ("%s @ L%d : incr : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + } \ + wc_debug_vector_registers_retval = (x); \ + } while (0) + +#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING + #ifndef WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED + #define WC_DEBUG_VECTOR_REGISTERS_FUZZING_SEED 0 + #endif + WOLFSSL_LOCAL int SAVE_VECTOR_REGISTERS2_fuzzer(void); + + #define SAVE_VECTOR_REGISTERS2(...) ({ \ + int _svr2_val = SAVE_VECTOR_REGISTERS2_fuzzer(); \ + if (_svr2_val == 0) { \ + ++wc_svr_count; \ + if (wc_svr_count > 5) { \ + fprintf(stderr, \ + ("%s @ L%d : incr : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + } \ + wc_svr_last_file = __FILE__; \ + wc_svr_last_line = __LINE__; \ + _svr2_val = 0; \ + } \ + _svr2_val; \ + }) + +#else + + #define SAVE_VECTOR_REGISTERS2(...) ({ \ + int _svr2_val; \ + if (wc_debug_vector_registers_retval != 0) { \ + if (wc_svr_count > 0) { \ + fprintf(stderr, \ + ("%s @ L%d : incr : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + } \ + _svr2_val = wc_debug_vector_registers_retval; \ + } else { \ + ++wc_svr_count; \ + if (wc_svr_count > 5) { \ + fprintf(stderr, \ + ("%s @ L%d : incr : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + } \ + wc_svr_last_file = __FILE__; \ + wc_svr_last_line = __LINE__; \ + _svr2_val = 0; \ + } \ + _svr2_val; \ + }) + +#endif + + #define ASSERT_SAVED_VECTOR_REGISTERS(fail_clause) do { \ + if (wc_svr_count <= 0) { \ + fprintf(stderr, \ + ("ASSERT_SAVED_VECTOR_REGISTERS : %s @ L%d : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + { fail_clause } \ + } \ + } while (0) + #define ASSERT_RESTORED_VECTOR_REGISTERS(fail_clause) do { \ + if (wc_svr_count != 0) { \ + fprintf(stderr, \ + ("ASSERT_RESTORED_VECTOR_REGISTERS : %s @ L%d" \ + " : wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + { fail_clause } \ + } \ + } while (0) + #define RESTORE_VECTOR_REGISTERS(...) do { \ + --wc_svr_count; \ + if ((wc_svr_count > 4) || (wc_svr_count < 0)) { \ + fprintf(stderr, \ + ("%s @ L%d : decr : " \ + "wc_svr_count %d (last op %s L%d)\n"), \ + __FILE__, \ + __LINE__, \ + wc_svr_count, \ + wc_svr_last_file, \ + wc_svr_last_line); \ + DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ + } \ + wc_svr_last_file = __FILE__; \ + wc_svr_last_line = __LINE__; \ + } while(0) +#endif + #ifdef __cplusplus } /* extern "C" */ #endif #endif /* WOLFSSL_MEMORY_H */ - diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 18d39336fb..dd378e28be 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -1566,90 +1566,24 @@ typedef struct w64wrapper { #define PRAGMA_DIAG_POP /* null expansion */ #endif - #ifdef DEBUG_VECTOR_REGISTER_ACCESS - WOLFSSL_API extern THREAD_LS_T int wc_svr_count; - WOLFSSL_API extern THREAD_LS_T const char *wc_svr_last_file; - WOLFSSL_API extern THREAD_LS_T int wc_svr_last_line; - - #ifdef DEBUG_VECTOR_REGISTERS_ABORT_ON_FAIL - #define DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE abort(); - #elif defined(DEBUG_VECTOR_REGISTERS_EXIT_ON_FAIL) - #define DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE exit(1); - #else - #define DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE - #endif - - #define SAVE_VECTOR_REGISTERS(...) { \ - ++wc_svr_count; \ - if (wc_svr_count > 5) { \ - fprintf(stderr, \ - "%s @ L%d : incr : wc_svr_count %d (last op %s L%d)\n", \ - __FILE__, \ - __LINE__, \ - wc_svr_count, \ - wc_svr_last_file, \ - wc_svr_last_line); \ - DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ - } \ - wc_svr_last_file = __FILE__; \ - wc_svr_last_line = __LINE__; \ - } - #define ASSERT_SAVED_VECTOR_REGISTERS(fail_clause) { \ - if (wc_svr_count <= 0) { \ - fprintf(stderr, \ - "ASSERT_SAVED_VECTOR_REGISTERS : %s @ L%d : wc_svr_count %d (last op %s L%d)\n", \ - __FILE__, \ - __LINE__, \ - wc_svr_count, \ - wc_svr_last_file, \ - wc_svr_last_line); \ - DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ - { fail_clause } \ - } \ - } - #define ASSERT_RESTORED_VECTOR_REGISTERS(fail_clause) { \ - if (wc_svr_count != 0) { \ - fprintf(stderr, \ - "ASSERT_RESTORED_VECTOR_REGISTERS : %s @ L%d : wc_svr_count %d (last op %s L%d)\n", \ - __FILE__, \ - __LINE__, \ - wc_svr_count, \ - wc_svr_last_file, \ - wc_svr_last_line); \ - DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ - { fail_clause } \ - } \ - } - #define RESTORE_VECTOR_REGISTERS(...) { \ - --wc_svr_count; \ - if ((wc_svr_count > 4) || (wc_svr_count < 0)) { \ - fprintf(stderr, \ - "%s @ L%d : decr : wc_svr_count %d (last op %s L%d)\n", \ - __FILE__, \ - __LINE__, \ - wc_svr_count, \ - wc_svr_last_file, \ - wc_svr_last_line); \ - DEBUG_VECTOR_REGISTERS_EXTRA_FAIL_CLAUSE \ - } \ - wc_svr_last_file = __FILE__; \ - wc_svr_last_line = __LINE__; \ - } - #else - #ifndef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(...) WC_DO_NOTHING - #endif - #ifndef ASSERT_SAVED_VECTOR_REGISTERS - #define ASSERT_SAVED_VECTOR_REGISTERS(...) WC_DO_NOTHING - #endif - #ifndef ASSERT_RESTORED_VECTOR_REGISTERS - #define ASSERT_RESTORED_VECTOR_REGISTERS(...) WC_DO_NOTHING - #endif - #ifndef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING - #endif + #ifndef SAVE_VECTOR_REGISTERS + #define SAVE_VECTOR_REGISTERS(...) WC_DO_NOTHING + #endif + #ifndef SAVE_VECTOR_REGISTERS2 + #define SAVE_VECTOR_REGISTERS2() 0 + #endif + #ifndef WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL + #define WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(x) WC_DO_NOTHING + #endif + #ifndef ASSERT_SAVED_VECTOR_REGISTERS + #define ASSERT_SAVED_VECTOR_REGISTERS(...) WC_DO_NOTHING + #endif + #ifndef ASSERT_RESTORED_VECTOR_REGISTERS + #define ASSERT_RESTORED_VECTOR_REGISTERS(...) WC_DO_NOTHING + #endif + #ifndef RESTORE_VECTOR_REGISTERS + #define RESTORE_VECTOR_REGISTERS() WC_DO_NOTHING #endif - #if FIPS_VERSION_GE(5,1) #define WC_SPKRE_F(x,y) wolfCrypt_SetPrivateKeyReadEnable_fips((x),(y))