From fa104be35e24f3fea895d55bb7042d6f4b2963e9 Mon Sep 17 00:00:00 2001
From: Andy Polyakov <appro@openssl.org>
Date: Thu, 10 Oct 2013 23:06:43 +0200
Subject: [PATCH] bn/asm/rsax-avx2.pl: minor optimization [for Decoded ICache].

---
 crypto/bn/asm/rsaz-avx2.pl | 151 ++++++++++++++++++-------------------
 1 file changed, 72 insertions(+), 79 deletions(-)

diff --git a/crypto/bn/asm/rsaz-avx2.pl b/crypto/bn/asm/rsaz-avx2.pl
index 2d9e239912..09c45b0ec2 100755
--- a/crypto/bn/asm/rsaz-avx2.pl
+++ b/crypto/bn/asm/rsaz-avx2.pl
@@ -48,7 +48,7 @@
 # +10% improvement by <appro@openssl.org>
 #
 # rsa2048 sign/sec	OpenSSL 1.0.1	scalar(*)	this
-# 2GHz Haswell		544		632/+16%	947/+74%
+# 2.3GHz Haswell	621		732/+18%	1112/+79%
 #
 # (*)	if system doesn't support AVX2, for reference purposes;
 
@@ -125,6 +125,7 @@ my $FrameSize=32*18+32*8;	# place for A^2 and 2*A
 my $aap=$r0;
 my $tp0="%rbx";
 my $tp1=$r3;
+my $tpa=$tmp;
 
 $np="%r13";			# reassigned argument
 
@@ -263,7 +264,7 @@ $code.=<<___;
 	 vpbroadcastq	32*2-128($ap), $B1
 	 vmovdqu	$ACC9, 32*17-448($tp1)
 
-	xor	$tmp, $tmp
+	mov	$ap, $tpa
 	mov 	\$4, $i
 	jmp	.Lsqr_entry_1024
 ___
@@ -272,38 +273,29 @@ $TEMP2=$Y2;
 $code.=<<___;
 .align	32
 .LOOP_SQR_1024:
-	vmovdqu		32*0(%rsp,$tmp), $TEMP0	# 32*0-192($tp0,$tmp)
-	vmovdqu		32*1(%rsp,$tmp), $TEMP1	# 32*1-192($tp0,$tmp)
-	 vpbroadcastq	32*1-128($ap,$tmp), $B2
+	 vpbroadcastq	32*1-128($tpa), $B2
 	vpmuludq	32*0-128($ap), $B1, $ACC0
-	vmovdqu		32*2-192($tp0,$tmp), $TEMP2
-	vpaddq		$TEMP0, $ACC0, $ACC0
+	vpaddq		32*0-192($tp0), $ACC0, $ACC0
 	vpmuludq	32*0-128($aap), $B1, $ACC1
-	vmovdqu		32*3-192($tp0,$tmp), $TEMP0
-	vpaddq		$TEMP1, $ACC1, $ACC1
+	vpaddq		32*1-192($tp0), $ACC1, $ACC1
 	vpmuludq	32*1-128($aap), $B1, $ACC2
-	vmovdqu		32*4-192($tp0,$tmp), $TEMP1
-	vpaddq		$TEMP2, $ACC2, $ACC2
+	vpaddq		32*2-192($tp0), $ACC2, $ACC2
 	vpmuludq	32*2-128($aap), $B1, $ACC3
-	vmovdqu		32*5-192($tp0,$tmp), $TEMP2
-	vpaddq		$TEMP0, $ACC3, $ACC3
+	vpaddq		32*3-192($tp0), $ACC3, $ACC3
 	vpmuludq	32*3-128($aap), $B1, $ACC4
-	vmovdqu		32*6-192($tp0,$tmp), $TEMP0
-	vpaddq		$TEMP1, $ACC4, $ACC4
+	vpaddq		32*4-192($tp0), $ACC4, $ACC4
 	vpmuludq	32*4-128($aap), $B1, $ACC5
-	vmovdqu		32*7-192($tp0,$tmp), $TEMP1
-	vpaddq		$TEMP2, $ACC5, $ACC5
+	vpaddq		32*5-192($tp0), $ACC5, $ACC5
 	vpmuludq	32*5-128($aap), $B1, $ACC6
-	vmovdqu		32*8-192($tp0,$tmp), $TEMP2
-	vpaddq		$TEMP0, $ACC6, $ACC6
+	vpaddq		32*6-192($tp0), $ACC6, $ACC6
 	vpmuludq	32*6-128($aap), $B1, $ACC7
-	vpaddq		$TEMP1, $ACC7, $ACC7
+	vpaddq		32*7-192($tp0), $ACC7, $ACC7
 	vpmuludq	32*7-128($aap), $B1, $ACC8
-	 vpbroadcastq	32*2-128($ap,$tmp), $B1
-	vpaddq		$TEMP2, $ACC8, $ACC8
+	 vpbroadcastq	32*2-128($tpa), $B1
+	vpaddq		32*8-192($tp0), $ACC8, $ACC8
 .Lsqr_entry_1024:
-	vmovdqu		$ACC0, 32*0(%rsp,$tmp)	# 32*0-192($tp0,$tmp)
-	vmovdqu		$ACC1, 32*1(%rsp,$tmp)	# 32*1-192($tp0,$tmp)
+	vmovdqu		$ACC0, 32*0-192($tp0)
+	vmovdqu		$ACC1, 32*1-192($tp0)
 
 	vpmuludq	32*1-128($ap), $B2, $TEMP0
 	vpaddq		$TEMP0, $ACC2, $ACC2
@@ -316,16 +308,15 @@ $code.=<<___;
 	vpmuludq	32*4-128($aap), $B2, $TEMP1
 	vpaddq		$TEMP1, $ACC6, $ACC6
 	vpmuludq	32*5-128($aap), $B2, $TEMP2
-	vmovdqu		32*9-192($tp0,$tmp), $TEMP1
 	vpaddq		$TEMP2, $ACC7, $ACC7
 	vpmuludq	32*6-128($aap), $B2, $TEMP0
 	vpaddq		$TEMP0, $ACC8, $ACC8
 	vpmuludq	32*7-128($aap), $B2, $ACC0
-	 vpbroadcastq	32*3-128($ap,$tmp), $B2
-	vpaddq		$TEMP1, $ACC0, $ACC0
+	 vpbroadcastq	32*3-128($tpa), $B2
+	vpaddq		32*9-192($tp0), $ACC0, $ACC0
 
-	vmovdqu		$ACC2, 32*2-192($tp0,$tmp)
-	vmovdqu		$ACC3, 32*3-192($tp0,$tmp)
+	vmovdqu		$ACC2, 32*2-192($tp0)
+	vmovdqu		$ACC3, 32*3-192($tp0)
 
 	vpmuludq	32*2-128($ap), $B1, $TEMP2
 	vpaddq		$TEMP2, $ACC4, $ACC4
@@ -336,16 +327,15 @@ $code.=<<___;
 	vpmuludq	32*4-128($aap), $B1, $TEMP2
 	vpaddq		$TEMP2, $ACC7, $ACC7
 	vpmuludq	32*5-128($aap), $B1, $TEMP0
-	vmovdqu		32*10-448($tp1,$tmp), $TEMP2
 	vpaddq		$TEMP0, $ACC8, $ACC8
 	vpmuludq	32*6-128($aap), $B1, $TEMP1
 	vpaddq		$TEMP1, $ACC0, $ACC0
 	vpmuludq	32*7-128($aap), $B1, $ACC1
-	 vpbroadcastq	32*4-128($ap,$tmp), $B1
-	vpaddq		$TEMP2, $ACC1, $ACC1
+	 vpbroadcastq	32*4-128($tpa), $B1
+	vpaddq		32*10-448($tp1), $ACC1, $ACC1
 
-	vmovdqu		$ACC4, 32*4-192($tp0,$tmp)
-	vmovdqu		$ACC5, 32*5-192($tp0,$tmp)
+	vmovdqu		$ACC4, 32*4-192($tp0)
+	vmovdqu		$ACC5, 32*5-192($tp0)
 
 	vpmuludq	32*3-128($ap), $B2, $TEMP0
 	vpaddq		$TEMP0, $ACC6, $ACC6
@@ -354,74 +344,70 @@ $code.=<<___;
 	vpmuludq	32*4-128($aap), $B2, $TEMP2
 	vpaddq		$TEMP2, $ACC8, $ACC8
 	vpmuludq	32*5-128($aap), $B2, $TEMP0
-	vmovdqu		32*11-448($tp1,$tmp), $TEMP2
 	vpaddq		$TEMP0, $ACC0, $ACC0
 	vpmuludq	32*6-128($aap), $B2, $TEMP1
 	vpaddq		$TEMP1, $ACC1, $ACC1
 	vpmuludq	32*7-128($aap), $B2, $ACC2
-	 vpbroadcastq	32*5-128($ap,$tmp), $B2
-	vpaddq		$TEMP2, $ACC2, $ACC2	
+	 vpbroadcastq	32*5-128($tpa), $B2
+	vpaddq		32*11-448($tp1), $ACC2, $ACC2	
 
-	vmovdqu		$ACC6, 32*6-192($tp0,$tmp)
-	vmovdqu		$ACC7, 32*7-192($tp0,$tmp)
+	vmovdqu		$ACC6, 32*6-192($tp0)
+	vmovdqu		$ACC7, 32*7-192($tp0)
 
 	vpmuludq	32*4-128($ap), $B1, $TEMP0
 	vpaddq		$TEMP0, $ACC8, $ACC8
 	vpmuludq	32*4-128($aap), $B1, $TEMP1
 	vpaddq		$TEMP1, $ACC0, $ACC0
 	vpmuludq	32*5-128($aap), $B1, $TEMP2
-	vmovdqu		32*12-448($tp1,$tmp), $TEMP1
 	vpaddq		$TEMP2, $ACC1, $ACC1
 	vpmuludq	32*6-128($aap), $B1, $TEMP0
 	vpaddq		$TEMP0, $ACC2, $ACC2
 	vpmuludq	32*7-128($aap), $B1, $ACC3
-	 vpbroadcastq	32*6-128($ap,$tmp), $B1
-	vpaddq		$TEMP1, $ACC3, $ACC3
+	 vpbroadcastq	32*6-128($tpa), $B1
+	vpaddq		32*12-448($tp1), $ACC3, $ACC3
 
-	vmovdqu		$ACC8, 32*8-192($tp0,$tmp)
-	vmovdqu		$ACC0, 32*9-192($tp0,$tmp)
+	vmovdqu		$ACC8, 32*8-192($tp0)
+	vmovdqu		$ACC0, 32*9-192($tp0)
+	lea		8($tp0), $tp0
 
 	vpmuludq	32*5-128($ap), $B2, $TEMP2
 	vpaddq		$TEMP2, $ACC1, $ACC1
 	vpmuludq	32*5-128($aap), $B2, $TEMP0
-	vmovdqu		32*13-448($tp1,$tmp), $TEMP2
 	vpaddq		$TEMP0, $ACC2, $ACC2
 	vpmuludq	32*6-128($aap), $B2, $TEMP1
 	vpaddq		$TEMP1, $ACC3, $ACC3
 	vpmuludq	32*7-128($aap), $B2, $ACC4
-	 vpbroadcastq	32*7-128($ap,$tmp), $B2
-	vpaddq		$TEMP2, $ACC4, $ACC4
+	 vpbroadcastq	32*7-128($tpa), $B2
+	vpaddq		32*13-448($tp1), $ACC4, $ACC4
 
-	vmovdqu		$ACC1, 32*10-448($tp1,$tmp)
-	vmovdqu		$ACC2, 32*11-448($tp1,$tmp)
+	vmovdqu		$ACC1, 32*10-448($tp1)
+	vmovdqu		$ACC2, 32*11-448($tp1)
 
 	vpmuludq	32*6-128($ap), $B1, $TEMP0
-	vmovdqu		32*14-448($tp1,$tmp), $TEMP2
 	vpaddq		$TEMP0, $ACC3, $ACC3
 	vpmuludq	32*6-128($aap), $B1, $TEMP1
-	 vpbroadcastq	32*8-128($ap,$tmp), $ACC0	# borrow $ACC0 for $B1
+	 vpbroadcastq	32*8-128($tpa), $ACC0		# borrow $ACC0 for $B1
 	vpaddq		$TEMP1, $ACC4, $ACC4
 	vpmuludq	32*7-128($aap), $B1, $ACC5
-	 vpbroadcastq	32*0+8-128($ap,$tmp), $B1	# for next iteration
-	vpaddq		$TEMP2, $ACC5, $ACC5
-	vmovdqu		32*15-448($tp1,$tmp), $TEMP1
+	 vpbroadcastq	32*0+8-128($tpa), $B1		# for next iteration
+	vpaddq		32*14-448($tp1), $ACC5, $ACC5
 
-	vmovdqu		$ACC3, 32*12-448($tp1,$tmp)
-	vmovdqu		$ACC4, 32*13-448($tp1,$tmp)
+	vmovdqu		$ACC3, 32*12-448($tp1)
+	vmovdqu		$ACC4, 32*13-448($tp1)
+	lea		8($tpa), $tpa
 
 	vpmuludq	32*7-128($ap), $B2, $TEMP0
-	vmovdqu		32*16-448($tp1,$tmp), $TEMP2
 	vpaddq		$TEMP0, $ACC5, $ACC5
 	vpmuludq	32*7-128($aap), $B2, $ACC6
-	vpaddq		$TEMP1, $ACC6, $ACC6
+	vpaddq		32*15-448($tp1), $ACC6, $ACC6
 
 	vpmuludq	32*8-128($ap), $ACC0, $ACC7
-	vmovdqu		$ACC5, 32*14-448($tp1,$tmp)
-	vpaddq		$TEMP2, $ACC7, $ACC7
-	vmovdqu		$ACC6, 32*15-448($tp1,$tmp)
-	vmovdqu		$ACC7, 32*16-448($tp1,$tmp)
+	vmovdqu		$ACC5, 32*14-448($tp1)
+	vpaddq		32*16-448($tp1), $ACC7, $ACC7
+	vmovdqu		$ACC6, 32*15-448($tp1)
+	vmovdqu		$ACC7, 32*16-448($tp1)
+	lea		8($tp1), $tp1
 
-	lea	8($tmp), $tmp
 	dec	$i        
 	jnz	.LOOP_SQR_1024
 ___
@@ -432,9 +418,10 @@ $TEMP3 = $Y1;
 $TEMP4 = $Y2;
 $code.=<<___;
 	#we need to fix indexes 32-39 to avoid overflow
-	vmovdqu		32*8-192($tp0), $ACC8
-	vmovdqu		32*9-192($tp0), $ACC1
-	vmovdqu		32*10-448($tp1), $ACC2
+	vmovdqu		32*8(%rsp), $ACC8		# 32*8-192($tp0),
+	vmovdqu		32*9(%rsp), $ACC1		# 32*9-192($tp0)
+	vmovdqu		32*10(%rsp), $ACC2		# 32*10-192($tp0)
+	lea		192(%rsp), $tp0			# 64+128=192
 
 	vpsrlq		\$29, $ACC8, $TEMP1
 	vpand		$AND_MASK, $ACC8, $ACC8
@@ -452,7 +439,7 @@ $code.=<<___;
 	vpaddq		$TEMP1, $ACC1, $ACC1
 	vpaddq		$TEMP2, $ACC2, $ACC2
 	vmovdqu		$ACC1, 32*9-192($tp0)
-	vmovdqu		$ACC2, 32*10-448($tp1)
+	vmovdqu		$ACC2, 32*10-192($tp0)
 
 	mov	(%rsp), %rax
 	mov	8(%rsp), $r1
@@ -502,13 +489,15 @@ $code.=<<___;
 	 mov	%rax, %rdx
 	 imulq	-128($np), %rax
 	vpaddq		$TEMP0, $ACC1, $ACC1
-	vpmuludq	32*2-128($np), $Y1, $TEMP1
 	 add	%rax, $r1
+	vpmuludq	32*2-128($np), $Y1, $TEMP1
 	 mov	%rdx, %rax
 	 imulq	8-128($np), %rax
 	vpaddq		$TEMP1, $ACC2, $ACC2
 	vpmuludq	32*3-128($np), $Y1, $TEMP2
+	 .byte	0x67
 	 add	%rax, $r2
+	 .byte	0x67
 	 mov	%rdx, %rax
 	 imulq	16-128($np), %rax
 	 shr	\$29, $r1
@@ -528,17 +517,17 @@ $code.=<<___;
 	vpaddq		$TEMP0, $ACC7, $ACC7
 	vpmuludq	32*8-128($np), $Y1, $TEMP1
 	 vmovd	%eax, $Y1
-	 vmovdqu	32*1-8-128($np), $TEMP2
+	 #vmovdqu	32*1-8-128($np), $TEMP2		# moved below
 	vpaddq		$TEMP1, $ACC8, $ACC8
-	 vmovdqu	32*2-8-128($np), $TEMP0
+	 #vmovdqu	32*2-8-128($np), $TEMP0		# moved below
 	 vpbroadcastq	$Y1, $Y1
 
-	vpmuludq	$Y2, $TEMP2, $TEMP2
+	vpmuludq	32*1-8-128($np), $Y2, $TEMP2	# see above
 	vmovdqu		32*3-8-128($np), $TEMP1
 	 mov	%rax, %rdx
 	 imulq	-128($np), %rax
 	vpaddq		$TEMP2, $ACC1, $ACC1
-	vpmuludq	$Y2, $TEMP0, $TEMP0
+	vpmuludq	32*2-8-128($np), $Y2, $TEMP0	# see above
 	vmovdqu		32*4-8-128($np), $TEMP2
 	 add	%rax, $r2
 	 mov	%rdx, %rax
@@ -552,11 +541,12 @@ $code.=<<___;
 	vpaddq		$TEMP1, $ACC3, $ACC3
 	vpmuludq	$Y2, $TEMP2, $TEMP2
 	vmovdqu		32*6-8-128($np), $TEMP1
+	 .byte	0x67
 	 mov	%rax, $r3
 	 imull	$n0, %eax
 	vpaddq		$TEMP2, $ACC4, $ACC4
 	vpmuludq	$Y2, $TEMP0, $TEMP0
-	vmovdqu		32*7-8-128($np), $TEMP2
+	.byte	0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00	# vmovdqu		32*7-8-128($np), $TEMP2
 	 and	\$0x1fffffff, %eax
 	vpaddq		$TEMP0, $ACC5, $ACC5
 	vpmuludq	$Y2, $TEMP1, $TEMP1
@@ -584,11 +574,12 @@ $code.=<<___;
 	vpaddq		$TEMP1, $ACC1, $ACC1
 	 vpmuludq	$Y2, $ACC0, $ACC0
 	vpmuludq	$Y1, $TEMP2, $TEMP2
-	vmovdqu		32*4-16-128($np), $TEMP1
+	.byte	0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff	# vmovdqu		32*4-16-128($np), $TEMP1
 	 vpaddq		$ACC1, $ACC0, $ACC0
 	vpaddq		$TEMP2, $ACC2, $ACC2
 	vpmuludq	$Y1, $TEMP0, $TEMP0
 	vmovdqu		32*5-16-128($np), $TEMP2
+	 .byte	0x67
 	 vmovq		$ACC0, %rax
 	 vmovdqu	$ACC0, (%rsp)		# transfer $r0-$r3
 	vpaddq		$TEMP0, $ACC3, $ACC3
@@ -602,12 +593,12 @@ $code.=<<___;
 	vmovdqu		32*8-16-128($np), $TEMP2
 	vpaddq		$TEMP0, $ACC6, $ACC6
 	vpmuludq	$Y1, $TEMP1, $TEMP1
-	vmovdqu		32*9-16-128($np), $TEMP0
 	 shr	\$29, $r3
+	vmovdqu		32*9-16-128($np), $TEMP0
+	 add	$r3, %rax
 	vpaddq		$TEMP1, $ACC7, $ACC7
 	vpmuludq	$Y1, $TEMP2, $TEMP2
-	 vmovdqu	32*2-24-128($np), $TEMP1
-	 add	$r3, %rax
+	 #vmovdqu	32*2-24-128($np), $TEMP1	# moved below
 	 mov	%rax, $r0
 	 imull	$n0, %eax
 	vpaddq		$TEMP2, $ACC8, $ACC8
@@ -615,10 +606,11 @@ $code.=<<___;
 	 and	\$0x1fffffff, %eax
 	 vmovd	%eax, $Y1
 	 vmovdqu	32*3-24-128($np), $TEMP2
+	.byte	0x67
 	vpaddq		$TEMP0, $ACC9, $ACC9
 	 vpbroadcastq	$Y1, $Y1
 
-	vpmuludq	$Y2, $TEMP1, $TEMP1
+	vpmuludq	32*2-24-128($np), $Y2, $TEMP1	# see above
 	vmovdqu		32*4-24-128($np), $TEMP0
 	 mov	%rax, %rdx
 	 imulq	-128($np), %rax
@@ -629,6 +621,7 @@ $code.=<<___;
 	 add	%rax, $r0
 	 mov	%rdx, %rax
 	 imulq	8-128($np), %rax
+	 .byte	0x67
 	 shr	\$29, $r0
 	 mov	16(%rsp), $r2
 	vpaddq		$TEMP2, $ACC3, $ACC2
@@ -648,8 +641,8 @@ $code.=<<___;
 	vmovdqu		32*8-24-128($np), $TEMP1
 	 mov	%rax, $r1
 	 imull	$n0, %eax
-	vpaddq		$TEMP2, $ACC6, $ACC5
 	vpmuludq	$Y2, $TEMP0, $TEMP0
+	vpaddq		$TEMP2, $ACC6, $ACC5
 	vmovdqu		32*9-24-128($np), $TEMP2
 	 and	\$0x1fffffff, %eax
 	vpaddq		$TEMP0, $ACC7, $ACC6