Upstream: Add Intel Indirect Branch Tracking support (#1659)

### Issues: * Addresses: CryptoAlg-2220 ### Previous Work: This change re-introduces changes from the following PRs: * #1628 Due to build failures with certain environments using gcc-4.8, the above was reverted: * #1656 The build failures were addressed in the following PR: * #1665 ### Description of changes: Upstream commits: * google/boringssl@9fc1c33 * google/boringssl@51ed32f By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license and the ISC license. --------- Co-authored-by: Bob Beck <[email protected]>
aws · Jul 12, 2024 · 1fbf584 · 1fbf584
1 parent fa8b98e
commit 1fbf584
Show file tree

Hide file tree

Showing 82 changed files with 469 additions and 0 deletions.
diff --git a/crypto/chacha/asm/chacha-x86_64.pl b/crypto/chacha/asm/chacha-x86_64.pl
@@ -235,6 +235,7 @@ sub ROUND {			# critical path is 24 cycles per round
 .align	64
 ChaCha20_ctr32:
 .cfi_startproc
+	_CET_ENDBR
 	cmp	\$0,$len
 	je	.Lno_data
 	mov	OPENSSL_ia32cap_P+4(%rip),%r10

diff --git a/crypto/cipher_extra/asm/aes128gcmsiv-x86_64.pl b/crypto/cipher_extra/asm/aes128gcmsiv-x86_64.pl
@@ -140,6 +140,7 @@ sub aesgcmsiv_htable_init {
 .align 16
 aesgcmsiv_htable_init:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqa ($H), $T
     vmovdqa $T, $TMP0
     vmovdqa $T, ($Htbl)      # H
@@ -180,6 +181,7 @@ sub aesgcmsiv_htable6_init {
 .align 16
 aesgcmsiv_htable6_init:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqa ($H), $T
     vmovdqa $T, $TMP0
     vmovdqa $T, ($Htbl)      # H
@@ -241,6 +243,7 @@ sub aesgcmsiv_htable_polyval {
 .align 16
 aesgcmsiv_htable_polyval:
 .cfi_startproc
+    _CET_ENDBR
     test  $len, $len
     jnz   .Lhtable_polyval_start
     ret
@@ -426,6 +429,7 @@ sub aesgcmsiv_polyval_horner {
 .align 16
 aesgcmsiv_polyval_horner:
 .cfi_startproc
+    _CET_ENDBR
     test $L, $L
     jnz .Lpolyval_horner_start
     ret
@@ -466,6 +470,7 @@ sub aesgcmsiv_polyval_horner {
 .align 16
 aes128gcmsiv_aes_ks:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqu (%rdi), %xmm1           # xmm1 = user key
     vmovdqa %xmm1, (%rsi)           # rsi points to output
 
@@ -527,6 +532,7 @@ sub aesgcmsiv_polyval_horner {
 .align 16
 aes256gcmsiv_aes_ks:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqu (%rdi), %xmm1
     vmovdqu 16(%rdi), %xmm3
     vmovdqa %xmm1, (%rsi)
@@ -620,6 +626,7 @@ sub aes128gcmsiv_aes_ks_enc_x1 {
 .align 16
 aes128gcmsiv_aes_ks_enc_x1:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqa (%rcx), %xmm1                 # xmm1 = first 16 bytes of random key
     vmovdqa 0*16(%rdi), $BLOCK1
 
@@ -693,6 +700,7 @@ sub aes128gcmsiv_kdf {
 .align 16
 aes128gcmsiv_kdf:
 .cfi_startproc
+    _CET_ENDBR
 # parameter 1: %rdi                         Pointer to NONCE
 # parameter 2: %rsi                         Pointer to CT
 # parameter 4: %rdx                         Pointer to keys
@@ -793,6 +801,7 @@ sub aes128gcmsiv_enc_msg_x4 {
 .align 16
 aes128gcmsiv_enc_msg_x4:
 .cfi_startproc
+    _CET_ENDBR
     test $LEN, $LEN
     jnz .L128_enc_msg_x4_start
     ret
@@ -990,6 +999,7 @@ sub aes128gcmsiv_enc_msg_x8 {
 .align 16
 aes128gcmsiv_enc_msg_x8:
 .cfi_startproc
+    _CET_ENDBR
     test $LEN, $LEN
     jnz .L128_enc_msg_x8_start
     ret
@@ -1245,6 +1255,7 @@ sub aesgcmsiv_dec {
 
   $code.=<<___;
 .cfi_startproc
+    _CET_ENDBR
     test \$~15, $LEN
     jnz .L${labelPrefix}_dec_start
     ret
@@ -1584,6 +1595,7 @@ sub aes128gcmsiv_ecb_enc_block {
 .align 16
 aes128gcmsiv_ecb_enc_block:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqa (%rdi), $STATE_1
 
     vpxor       ($KSp), $STATE_1, $STATE_1
@@ -1676,6 +1688,7 @@ sub aes256gcmsiv_aes_ks_enc_x1 {
 .align 16
 aes256gcmsiv_aes_ks_enc_x1:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqa con1(%rip), $CON_MASK    # CON_MASK  = 1,1,1,1
     vmovdqa mask(%rip), $MASK_256    # MASK_256
     vmovdqa ($PT), $BLOCK1
@@ -1717,6 +1730,7 @@ sub aes256gcmsiv_ecb_enc_block {
 .align 16
 aes256gcmsiv_ecb_enc_block:
 .cfi_startproc
+    _CET_ENDBR
     vmovdqa (%rdi), $STATE_1
     vpxor ($KSp), $STATE_1, $STATE_1
     vaesenc 1*16($KSp), $STATE_1, $STATE_1
@@ -1800,6 +1814,7 @@ sub aes256gcmsiv_enc_msg_x4 {
 .align 16
 aes256gcmsiv_enc_msg_x4:
 .cfi_startproc
+    _CET_ENDBR
     test $LEN, $LEN
     jnz .L256_enc_msg_x4_start
     ret
@@ -2000,6 +2015,7 @@ ()
 .align 16
 aes256gcmsiv_enc_msg_x8:
 .cfi_startproc
+    _CET_ENDBR
     test $LEN, $LEN
     jnz .L256_enc_msg_x8_start
     ret
@@ -2206,6 +2222,7 @@ sub aes256gcmsiv_kdf {
 .align 16
 aes256gcmsiv_kdf:
 .cfi_startproc
+    _CET_ENDBR
 # parameter 1: %rdi                         Pointer to NONCE
 # parameter 2: %rsi                         Pointer to CT
 # parameter 4: %rdx                         Pointer to keys

diff --git a/crypto/cipher_extra/asm/chacha20_poly1305_x86_64.pl b/crypto/cipher_extra/asm/chacha20_poly1305_x86_64.pl
@@ -453,6 +453,7 @@ sub emit_body {
 .align 64
 chacha20_poly1305_open:
 .cfi_startproc
+    _CET_ENDBR
     push %rbp
 .cfi_push %rbp
     push %rbx
@@ -875,6 +876,7 @@ sub emit_body {
 .align 64
 chacha20_poly1305_seal:
 .cfi_startproc
+    _CET_ENDBR
     push %rbp
 .cfi_push %rbp
     push %rbx

diff --git a/crypto/fipsmodule/aes/asm/aesni-x86_64.pl b/crypto/fipsmodule/aes/asm/aesni-x86_64.pl
@@ -278,6 +278,7 @@ sub aesni_generate1 {
 .align	16
 ${PREFIX}_encrypt:
 .cfi_startproc
+	_CET_ENDBR
 #ifdef BORINGSSL_DISPATCH_TEST
 .extern	BORINGSSL_function_hit
 	movb \$1,BORINGSSL_function_hit+1(%rip)
@@ -300,6 +301,7 @@ sub aesni_generate1 {
 .align	16
 ${PREFIX}_decrypt:
 .cfi_startproc
+	_CET_ENDBR
 	movups	($inp),$inout0		# load input
 	mov	240($key),$rounds	# key->rounds
 ___
@@ -620,6 +622,7 @@ sub aesni_generate8 {
 .align	16
 ${PREFIX}_ecb_encrypt:
 .cfi_startproc
+	_CET_ENDBR
 ___
 $code.=<<___ if ($win64);
 	lea	-0x58(%rsp),%rsp
@@ -1206,6 +1209,7 @@ sub aesni_generate8 {
 .align	16
 ${PREFIX}_ctr32_encrypt_blocks:
 .cfi_startproc
+	_CET_ENDBR
 #ifdef BORINGSSL_DISPATCH_TEST
 	movb \$1,BORINGSSL_function_hit(%rip)
 #endif
@@ -1784,6 +1788,7 @@ sub aesni_generate8 {
 .align	16
 ${PREFIX}_xts_encrypt:
 .cfi_startproc
+	_CET_ENDBR
 	lea	(%rsp),%r11			# frame pointer
 .cfi_def_cfa_register	%r11
 	push	%rbp
@@ -2267,6 +2272,7 @@ sub aesni_generate8 {
 .align	16
 ${PREFIX}_xts_decrypt:
 .cfi_startproc
+	_CET_ENDBR
 	lea	(%rsp),%r11			# frame pointer
 .cfi_def_cfa_register	%r11
 	push	%rbp
@@ -2785,6 +2791,7 @@ sub aesni_generate8 {
 .align	16
 ${PREFIX}_cbc_encrypt:
 .cfi_startproc
+	_CET_ENDBR
 	test	$len,$len		# check length
 	jz	.Lcbc_ret
 
@@ -3334,6 +3341,7 @@ sub aesni_generate8 {
 .align	16
 ${PREFIX}_set_decrypt_key:
 .cfi_startproc
+	_CET_ENDBR
 	.byte	0x48,0x83,0xEC,0x08	# sub rsp,8
 .cfi_adjust_cfa_offset	8
 	call	__aesni_set_encrypt_key
@@ -3406,6 +3414,7 @@ sub aesni_generate8 {
 ${PREFIX}_set_encrypt_key:
 __aesni_set_encrypt_key:
 .cfi_startproc
+	_CET_ENDBR
 #ifdef BORINGSSL_DISPATCH_TEST
 	movb \$1,BORINGSSL_function_hit+3(%rip)
 #endif

diff --git a/crypto/fipsmodule/aes/asm/vpaes-x86_64.pl b/crypto/fipsmodule/aes/asm/vpaes-x86_64.pl
@@ -874,6 +874,7 @@
 .align	16
 ${PREFIX}_set_encrypt_key:
 .cfi_startproc
+	_CET_ENDBR
 #ifdef BORINGSSL_DISPATCH_TEST
 .extern        BORINGSSL_function_hit
        movb \$1, BORINGSSL_function_hit+5(%rip)
@@ -929,6 +930,7 @@
 .align	16
 ${PREFIX}_set_decrypt_key:
 .cfi_startproc
+	_CET_ENDBR
 ___
 $code.=<<___ if ($win64);
 	lea	-0xb8(%rsp),%rsp
@@ -984,6 +986,7 @@
 .align	16
 ${PREFIX}_encrypt:
 .cfi_startproc
+	_CET_ENDBR
 #ifdef BORINGSSL_DISPATCH_TEST
 .extern        BORINGSSL_function_hit
        movb \$1, BORINGSSL_function_hit+4(%rip)
@@ -1033,6 +1036,7 @@
 .align	16
 ${PREFIX}_decrypt:
 .cfi_startproc
+	_CET_ENDBR
 ___
 $code.=<<___ if ($win64);
 	lea	-0xb8(%rsp),%rsp
@@ -1084,6 +1088,7 @@
 .align	16
 ${PREFIX}_cbc_encrypt:
 .cfi_startproc
+	_CET_ENDBR
 	xchg	$key,$len
 ___
 ($len,$key)=($key,$len);
@@ -1169,6 +1174,7 @@
 .align	16
 ${PREFIX}_ctr32_encrypt_blocks:
 .cfi_startproc
+	_CET_ENDBR
 	# _vpaes_encrypt_core and _vpaes_encrypt_core_2x expect the key in %rdx.
 	xchg	$key, $blocks
 ___

diff --git a/crypto/fipsmodule/bn/asm/rsaz-avx2.pl b/crypto/fipsmodule/bn/asm/rsaz-avx2.pl
@@ -116,6 +116,7 @@
 .align	64
 rsaz_1024_sqr_avx2:		# 702 cycles, 14% faster than rsaz_1024_mul_avx2
 .cfi_startproc
+	_CET_ENDBR
 	lea	(%rsp), %rax
 .cfi_def_cfa_register	%rax
 	push	%rbx
@@ -867,6 +868,7 @@
 .align	64
 rsaz_1024_mul_avx2:
 .cfi_startproc
+	_CET_ENDBR
 	lea	(%rsp), %rax
 .cfi_def_cfa_register	%rax
 	push	%rbx
@@ -1478,6 +1480,7 @@
 .align	32
 rsaz_1024_red2norm_avx2:
 .cfi_startproc
+	_CET_ENDBR
 	sub	\$-128,$inp	# size optimization
 	xor	%rax,%rax
 ___
@@ -1519,6 +1522,7 @@
 .align	32
 rsaz_1024_norm2red_avx2:
 .cfi_startproc
+	_CET_ENDBR
 	sub	\$-128,$out	# size optimization
 	mov	($inp),@T[0]
 	mov	\$0x1fffffff,%eax
@@ -1563,6 +1567,7 @@
 .align	32
 rsaz_1024_scatter5_avx2:
 .cfi_startproc
+	_CET_ENDBR
 	vzeroupper
 	vmovdqu	.Lscatter_permd(%rip),%ymm5
 	shl	\$4,$power
@@ -1590,6 +1595,7 @@
 .align	32
 rsaz_1024_gather5_avx2:
 .cfi_startproc
+	_CET_ENDBR
 	vzeroupper
 	mov	%rsp,%r11
 .cfi_def_cfa_register	%r11

diff --git a/crypto/fipsmodule/bn/asm/x86_64-mont.pl b/crypto/fipsmodule/bn/asm/x86_64-mont.pl
@@ -96,6 +96,7 @@
 .align	16
 bn_mul_mont:
 .cfi_startproc
+	_CET_ENDBR
 	mov	${num}d,${num}d
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax

diff --git a/crypto/fipsmodule/bn/asm/x86_64-mont5.pl b/crypto/fipsmodule/bn/asm/x86_64-mont5.pl
@@ -83,6 +83,7 @@
 .align	64
 bn_mul_mont_gather5:
 .cfi_startproc
+	_CET_ENDBR
 	mov	${num}d,${num}d
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
@@ -1106,6 +1107,7 @@
 .align	32
 bn_power5:
 .cfi_startproc
+	_CET_ENDBR
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
 ___
@@ -1250,6 +1252,7 @@
 bn_sqr8x_internal:
 __bn_sqr8x_internal:
 .cfi_startproc
+	_CET_ENDBR
 	##############################################################
 	# Squaring part:
 	#
@@ -2748,6 +2751,7 @@
 bn_sqrx8x_internal:
 __bn_sqrx8x_internal:
 .cfi_startproc
+	_CET_ENDBR
 	##################################################################
 	# Squaring part:
 	#
@@ -3460,6 +3464,7 @@
 .align	16
 bn_scatter5:
 .cfi_startproc
+	_CET_ENDBR
 	cmp	\$0, $num
 	jz	.Lscatter_epilogue
 
@@ -3490,6 +3495,7 @@
 bn_gather5:
 .cfi_startproc
 .LSEH_begin_bn_gather5:			# Win64 thing, but harmless in other cases
+	_CET_ENDBR
 	# I can't trust assembler to use specific encoding:-(
 	.byte	0x4c,0x8d,0x14,0x24			#lea    (%rsp),%r10
 .cfi_def_cfa_register	%r10