poly1305/asm/poly1305-x86_64.pl: switch to pure AVX512F.
Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be executed even on Knights Landing. Trigger for modification was observation that AVX512 code paths can negatively affect overall Skylake-X system performance. Since we are likely to suppress AVX512F capability flag [at least on Skylake-X], conversion serves as kind of "investment protection". Reviewed-by: Rich Salz <rsalz@openssl.org> (Merged from https://github.com/openssl/openssl/pull/4758)
This commit is contained in:
Andy Polyakov
parent
10a3195fcf
commit
a8f302e5ba
1 changed files with 72 additions and 68 deletions
|
|
@ -24,6 +24,16 @@
|
|||
#
|
||||
# Add AVX512F+VL+BW code path.
|
||||
#
|
||||
# November 2017
|
||||
#
|
||||
# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be
|
||||
# executed even on Knights Landing. Trigger for modification was
|
||||
# observation that AVX512 code paths can negatively affect overall
|
||||
# Skylake-X system performance. Since we are likely to suppress
|
||||
# AVX512F capability flag [at least on Skylake-X], conversion serves
|
||||
# as kind of "investment protection". Note that next *lake processor,
|
||||
# Cannolake, has AVX512IFMA code path to execute...
|
||||
#
|
||||
# Numbers are cycles per processed byte with poly1305_blocks alone,
|
||||
# measured with rdtsc at fixed clock frequency.
|
||||
#
|
||||
|
|
@ -35,7 +45,7 @@
|
|||
# Haswell 1.14/+175% 1.11 0.65
|
||||
# Skylake[-X] 1.13/+120% 0.96 0.51 [0.35]
|
||||
# Silvermont 2.83/+95% -
|
||||
# Knights L 3.60/- 1.65 1.10 (***)
|
||||
# Knights L 3.60/? 1.65 1.10 ?
|
||||
# Goldmont 1.70/+180% -
|
||||
# VIA Nano 1.82/+150% -
|
||||
# Sledgehammer 1.38/+160% -
|
||||
|
|
@ -50,8 +60,6 @@
|
|||
# Core processors, 50-30%, less newer processor is, but slower on
|
||||
# contemporary ones, for example almost 2x slower on Atom, and as
|
||||
# former are naturally disappearing, SSE2 is deemed unnecessary;
|
||||
# (***) Current AVX-512 code requires BW and VL extensions and can not
|
||||
# execute on Knights Landing;
|
||||
|
||||
$flavour = shift;
|
||||
$output = shift;
|
||||
|
|
@ -1685,7 +1693,6 @@ poly1305_blocks_avx2:
|
|||
.Leven_avx2:
|
||||
.cfi_startproc
|
||||
mov OPENSSL_ia32cap_P+8(%rip),%r10d
|
||||
mov \$`(1<<31|1<<30|1<<16)`,%r11d
|
||||
vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26
|
||||
vmovd 4*1($ctx),%x#$H1
|
||||
vmovd 4*2($ctx),%x#$H2
|
||||
|
|
@ -1698,8 +1705,8 @@ $code.=<<___ if ($avx>2);
|
|||
cmp \$512,$len
|
||||
jb .Lskip_avx512
|
||||
and %r11d,%r10d
|
||||
cmp %r11d,%r10d # check for AVX512F+BW+VL
|
||||
je .Lblocks_avx512
|
||||
test \$`1<<16`,%r10d # check for AVX512F
|
||||
jnz .Lblocks_avx512
|
||||
.Lskip_avx512:
|
||||
___
|
||||
$code.=<<___ if (!$win64);
|
||||
|
|
@ -2109,10 +2116,14 @@ if ($avx>2) {
|
|||
# reason stack layout is kept identical to poly1305_blocks_avx2. If not
|
||||
# for this tail, we wouldn't have to even allocate stack frame...
|
||||
|
||||
my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%ymm$_",(16..24));
|
||||
my ($M0,$M1,$M2,$M3,$M4) = map("%ymm$_",(25..29));
|
||||
my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24));
|
||||
my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29));
|
||||
my $PADBIT="%zmm30";
|
||||
my $GATHER="%ymm31";
|
||||
|
||||
map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain
|
||||
map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4));
|
||||
map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4));
|
||||
map(s/%y/%z/,($MASK));
|
||||
|
||||
$code.=<<___;
|
||||
.type poly1305_blocks_avx512,\@function,4
|
||||
|
|
@ -2120,7 +2131,8 @@ $code.=<<___;
|
|||
poly1305_blocks_avx512:
|
||||
.cfi_startproc
|
||||
.Lblocks_avx512:
|
||||
vzeroupper
|
||||
mov \$15,%eax
|
||||
kmovw %eax,%k2
|
||||
___
|
||||
$code.=<<___ if (!$win64);
|
||||
lea -8(%rsp),%r11
|
||||
|
|
@ -2133,52 +2145,53 @@ $code.=<<___ if ($win64);
|
|||
vmovdqa %xmm6,0x50(%r11)
|
||||
vmovdqa %xmm7,0x60(%r11)
|
||||
vmovdqa %xmm8,0x70(%r11)
|
||||
vmovdqa32 %xmm9,0x80(%r11)
|
||||
vmovdqa32 %xmm10,0x90(%r11)
|
||||
vmovdqa32 %xmm11,0xa0(%r11)
|
||||
vmovdqa32 %xmm12,0xb0(%r11)
|
||||
vmovdqa32 %xmm13,0xc0(%r11)
|
||||
vmovdqa32 %xmm14,0xd0(%r11)
|
||||
vmovdqa32 %xmm15,0xe0(%r11)
|
||||
vmovdqa %xmm9,0x80(%r11)
|
||||
vmovdqa %xmm10,0x90(%r11)
|
||||
vmovdqa %xmm11,0xa0(%r11)
|
||||
vmovdqa %xmm12,0xb0(%r11)
|
||||
vmovdqa %xmm13,0xc0(%r11)
|
||||
vmovdqa %xmm14,0xd0(%r11)
|
||||
vmovdqa %xmm15,0xe0(%r11)
|
||||
.Ldo_avx512_body:
|
||||
___
|
||||
$code.=<<___;
|
||||
lea .Lconst(%rip),%rcx
|
||||
lea 48+64($ctx),$ctx # size optimization
|
||||
vmovdqa 96(%rcx),$T2 # .Lpermd_avx2
|
||||
vmovdqa 96(%rcx),%y#$T2 # .Lpermd_avx2
|
||||
|
||||
# expand pre-calculated table
|
||||
vmovdqu32 `16*0-64`($ctx),%x#$R0
|
||||
vmovdqu32 `16*0-64`($ctx),${R0}{%k2}{z}
|
||||
and \$-512,%rsp
|
||||
vmovdqu32 `16*1-64`($ctx),%x#$R1
|
||||
vmovdqu32 `16*2-64`($ctx),%x#$S1
|
||||
vmovdqu32 `16*3-64`($ctx),%x#$R2
|
||||
vmovdqu32 `16*4-64`($ctx),%x#$S2
|
||||
vmovdqu32 `16*5-64`($ctx),%x#$R3
|
||||
vmovdqu32 `16*6-64`($ctx),%x#$S3
|
||||
vmovdqu32 `16*7-64`($ctx),%x#$R4
|
||||
vmovdqu32 `16*8-64`($ctx),%x#$S4
|
||||
vmovdqu32 `16*1-64`($ctx),${R1}{%k2}{z}
|
||||
mov \$0x20,%rax
|
||||
vmovdqu32 `16*2-64`($ctx),${S1}{%k2}{z}
|
||||
vmovdqu32 `16*3-64`($ctx),${R2}{%k2}{z}
|
||||
vmovdqu32 `16*4-64`($ctx),${S2}{%k2}{z}
|
||||
vmovdqu32 `16*5-64`($ctx),${R3}{%k2}{z}
|
||||
vmovdqu32 `16*6-64`($ctx),${S3}{%k2}{z}
|
||||
vmovdqu32 `16*7-64`($ctx),${R4}{%k2}{z}
|
||||
vmovdqu32 `16*8-64`($ctx),${S4}{%k2}{z}
|
||||
vpermd $R0,$T2,$R0 # 00003412 -> 14243444
|
||||
vmovdqa64 64(%rcx),$MASK # .Lmask26
|
||||
vpbroadcastq 64(%rcx),$MASK # .Lmask26
|
||||
vpermd $R1,$T2,$R1
|
||||
vpermd $S1,$T2,$S1
|
||||
vpermd $R2,$T2,$R2
|
||||
vmovdqa32 $R0,0x00(%rsp) # save in case $len%128 != 0
|
||||
vmovdqa64 $R0,0x00(%rsp){%k2} # save in case $len%128 != 0
|
||||
vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304
|
||||
vpermd $S2,$T2,$S2
|
||||
vmovdqa32 $R1,0x20(%rsp)
|
||||
vmovdqu64 $R1,0x00(%rsp,%rax){%k2}
|
||||
vpsrlq \$32,$R1,$T1
|
||||
vpermd $R3,$T2,$R3
|
||||
vmovdqa32 $S1,0x40(%rsp)
|
||||
vmovdqa64 $S1,0x40(%rsp){%k2}
|
||||
vpermd $S3,$T2,$S3
|
||||
vpermd $R4,$T2,$R4
|
||||
vmovdqa32 $R2,0x60(%rsp)
|
||||
vmovdqu64 $R2,0x40(%rsp,%rax){%k2}
|
||||
vpermd $S4,$T2,$S4
|
||||
vmovdqa32 $S2,0x80(%rsp)
|
||||
vmovdqa32 $R3,0xa0(%rsp)
|
||||
vmovdqa32 $S3,0xc0(%rsp)
|
||||
vmovdqa32 $R4,0xe0(%rsp)
|
||||
vmovdqa32 $S4,0x100(%rsp)
|
||||
vmovdqa64 $S2,0x80(%rsp){%k2}
|
||||
vmovdqu64 $R3,0x80(%rsp,%rax){%k2}
|
||||
vmovdqa64 $S3,0xc0(%rsp){%k2}
|
||||
vmovdqu64 $R4,0xc0(%rsp,%rax){%k2}
|
||||
vmovdqa64 $S4,0x100(%rsp){%k2}
|
||||
|
||||
################################################################
|
||||
# calculate 5th through 8th powers of the key
|
||||
|
|
@ -2282,14 +2295,6 @@ $code.=<<___;
|
|||
vpandq $MASK,$D3,$D3
|
||||
vpaddq $M3,$D4,$D4 # d3 -> d4
|
||||
|
||||
___
|
||||
map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain
|
||||
map(s/%y/%z/,($M4,$M0,$M1,$M2,$M3));
|
||||
map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4));
|
||||
map(s/%y/%z/,($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4));
|
||||
map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4));
|
||||
map(s/%y/%z/,($MASK));
|
||||
$code.=<<___;
|
||||
################################################################
|
||||
# at this point we have 14243444 in $R0-$S4 and 05060708 in
|
||||
# $D0-$D4, ...
|
||||
|
|
@ -2327,7 +2332,6 @@ $code.=<<___;
|
|||
vpaddd $R3,$S3,$S3
|
||||
vpaddd $R4,$S4,$S4
|
||||
|
||||
vpbroadcastq %x#$MASK,$MASK
|
||||
vpbroadcastq 32(%rcx),$PADBIT # .L129
|
||||
|
||||
vpsrlq \$52,$T0,$T2 # splat input
|
||||
|
|
@ -2345,7 +2349,7 @@ $code.=<<___;
|
|||
vpaddq $H2,$T2,$H2 # accumulate input
|
||||
sub \$192,$len
|
||||
jbe .Ltail_avx512
|
||||
#jmp .Loop_avx512
|
||||
jmp .Loop_avx512
|
||||
|
||||
.align 32
|
||||
.Loop_avx512:
|
||||
|
|
@ -2532,7 +2536,7 @@ $code.=<<___;
|
|||
vpaddq $H3,$T3,$H3
|
||||
vpaddq $H4,$T4,$H4
|
||||
|
||||
vmovdqu64 16*0($inp),%x#$T0
|
||||
vmovdqu 16*0($inp),%x#$T0
|
||||
vpmuludq $H0,$R3,$M3
|
||||
vpmuludq $H0,$R4,$M4
|
||||
vpmuludq $H0,$R0,$M0
|
||||
|
|
@ -2542,7 +2546,7 @@ $code.=<<___;
|
|||
vpaddq $M0,$D0,$D0 # d0 += h0*r0
|
||||
vpaddq $M1,$D1,$D1 # d1 += h0*r1
|
||||
|
||||
vmovdqu64 16*1($inp),%x#$T1
|
||||
vmovdqu 16*1($inp),%x#$T1
|
||||
vpmuludq $H1,$R2,$M3
|
||||
vpmuludq $H1,$R3,$M4
|
||||
vpmuludq $H1,$S4,$M0
|
||||
|
|
@ -2552,7 +2556,7 @@ $code.=<<___;
|
|||
vpaddq $M0,$D0,$D0 # d0 += h1*s4
|
||||
vpaddq $M2,$D2,$D2 # d2 += h0*r2
|
||||
|
||||
vinserti64x2 \$1,16*2($inp),$T0,$T0
|
||||
vinserti128 \$1,16*2($inp),%y#$T0,%y#$T0
|
||||
vpmuludq $H3,$R0,$M3
|
||||
vpmuludq $H3,$R1,$M4
|
||||
vpmuludq $H1,$R0,$M1
|
||||
|
|
@ -2562,7 +2566,7 @@ $code.=<<___;
|
|||
vpaddq $M1,$D1,$D1 # d1 += h1*r0
|
||||
vpaddq $M2,$D2,$D2 # d2 += h1*r1
|
||||
|
||||
vinserti64x2 \$1,16*3($inp),$T1,$T1
|
||||
vinserti128 \$1,16*3($inp),%y#$T1,%y#$T1
|
||||
vpmuludq $H4,$S4,$M3
|
||||
vpmuludq $H4,$R0,$M4
|
||||
vpmuludq $H3,$S2,$M0
|
||||
|
|
@ -2585,11 +2589,11 @@ $code.=<<___;
|
|||
# horizontal addition
|
||||
|
||||
mov \$1,%eax
|
||||
vpsrldq \$8,$H3,$D3
|
||||
vpsrldq \$8,$D4,$H4
|
||||
vpsrldq \$8,$H0,$D0
|
||||
vpsrldq \$8,$H1,$D1
|
||||
vpsrldq \$8,$H2,$D2
|
||||
vpermq \$0xb1,$H3,$D3
|
||||
vpermq \$0xb1,$D4,$H4
|
||||
vpermq \$0xb1,$H0,$D0
|
||||
vpermq \$0xb1,$H1,$D1
|
||||
vpermq \$0xb1,$H2,$D2
|
||||
vpaddq $D3,$H3,$H3
|
||||
vpaddq $D4,$H4,$H4
|
||||
vpaddq $D0,$H0,$H0
|
||||
|
|
@ -2626,23 +2630,23 @@ $code.=<<___;
|
|||
# lazy reduction (interleaved with input splat)
|
||||
|
||||
vpsrlq \$26,$H3,$D3
|
||||
vpandq $MASK,$H3,$H3
|
||||
vpand $MASK,$H3,$H3
|
||||
vpsrldq \$6,$T0,$T2 # splat input
|
||||
vpsrldq \$6,$T1,$T3
|
||||
vpunpckhqdq $T1,$T0,$T4 # 4
|
||||
vpaddq $D3,$H4,$H4 # h3 -> h4
|
||||
|
||||
vpsrlq \$26,$H0,$D0
|
||||
vpandq $MASK,$H0,$H0
|
||||
vpand $MASK,$H0,$H0
|
||||
vpunpcklqdq $T3,$T2,$T2 # 2:3
|
||||
vpunpcklqdq $T1,$T0,$T0 # 0:1
|
||||
vpaddq $D0,$H1,$H1 # h0 -> h1
|
||||
|
||||
vpsrlq \$26,$H4,$D4
|
||||
vpandq $MASK,$H4,$H4
|
||||
vpand $MASK,$H4,$H4
|
||||
|
||||
vpsrlq \$26,$H1,$D1
|
||||
vpandq $MASK,$H1,$H1
|
||||
vpand $MASK,$H1,$H1
|
||||
vpsrlq \$30,$T2,$T3
|
||||
vpsrlq \$4,$T2,$T2
|
||||
vpaddq $D1,$H2,$H2 # h1 -> h2
|
||||
|
|
@ -2654,21 +2658,21 @@ $code.=<<___;
|
|||
vpaddq $D4,$H0,$H0 # h4 -> h0
|
||||
|
||||
vpsrlq \$26,$H2,$D2
|
||||
vpandq $MASK,$H2,$H2
|
||||
vpandq $MASK,$T2,$T2 # 2
|
||||
vpandq $MASK,$T0,$T0 # 0
|
||||
vpand $MASK,$H2,$H2
|
||||
vpand $MASK,$T2,$T2 # 2
|
||||
vpand $MASK,$T0,$T0 # 0
|
||||
vpaddq $D2,$H3,$H3 # h2 -> h3
|
||||
|
||||
vpsrlq \$26,$H0,$D0
|
||||
vpandq $MASK,$H0,$H0
|
||||
vpand $MASK,$H0,$H0
|
||||
vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2
|
||||
vpandq $MASK,$T1,$T1 # 1
|
||||
vpand $MASK,$T1,$T1 # 1
|
||||
vpaddq $D0,$H1,$H1 # h0 -> h1
|
||||
|
||||
vpsrlq \$26,$H3,$D3
|
||||
vpandq $MASK,$H3,$H3
|
||||
vpandq $MASK,$T3,$T3 # 3
|
||||
vporq $PADBIT,$T4,$T4 # padbit, yes, always
|
||||
vpand $MASK,$H3,$H3
|
||||
vpand $MASK,$T3,$T3 # 3
|
||||
vpor 32(%rcx),$T4,$T4 # padbit, yes, always
|
||||
vpaddq $D3,$H4,$H4 # h3 -> h4
|
||||
|
||||
lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2
|
||||
|
|
|
|||
Loading…
Reference in a new issue