x86 assembler pack: add bn_GF2m_mul_2x2 implementations (see x86-gf2m.pl for

details and performance data).
This commit is contained in:
Andy Polyakov 2011-05-07 10:31:06 +00:00
parent 4fa3c4c3e9
commit 58cc21fdea
4 changed files with 419 additions and 34 deletions

View file

@ -123,7 +123,7 @@ my $tlib="-lnsl -lsocket";
my $bits1="THIRTY_TWO_BIT ";
my $bits2="SIXTY_FOUR_BIT ";
my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o:des-586.o crypt586.o:aes-586.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o";
my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o:des-586.o crypt586.o:aes-586.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o";
my $x86_elf_asm="$x86_asm:elf";

130
TABLE
View file

@ -233,7 +233,7 @@ $sys_id =
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -265,7 +265,7 @@ $sys_id =
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -329,7 +329,7 @@ $sys_id = CYGWIN32
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -393,7 +393,7 @@ $sys_id = MSDOS
$lflags = -L/dev/env/WATT_ROOT/lib -lwatt
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -553,7 +553,7 @@ $sys_id =
$lflags = -lsocket
$bn_ops = DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -745,7 +745,7 @@ $sys_id = WIN32
$lflags =
$bn_ops = BN_LLONG RC4_INDEX EXPORT_VAR_AS_FN RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1097,7 +1097,7 @@ $sys_id = BEOS
$lflags = -lbe -lbind -lsocket
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1129,7 +1129,7 @@ $sys_id = BEOS
$lflags = -lbe -lnet
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1161,7 +1161,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1289,7 +1289,7 @@ $sys_id = MACOSX
$lflags = -Wl,-search_paths_first%
$bn_ops = BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1449,7 +1449,7 @@ $sys_id =
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1513,7 +1513,7 @@ $sys_id = WIN32
$lflags =
$bn_ops = BN_LLONG RC4_INDEX EXPORT_VAR_AS_FN RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1801,7 +1801,7 @@ $sys_id =
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1833,7 +1833,7 @@ $sys_id = MACOSX
$lflags = -Wl,-search_paths_first%
$bn_ops = BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1961,7 +1961,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -1993,7 +1993,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2089,7 +2089,7 @@ $sys_id =
$lflags = -lefence -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2121,7 +2121,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2249,7 +2249,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2281,7 +2281,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2345,7 +2345,7 @@ $sys_id =
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2537,7 +2537,7 @@ $sys_id =
$lflags = -rdynamic -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -2729,7 +2729,7 @@ $sys_id =
$lflags = -lnsl -lsocket
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -3305,7 +3305,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -3657,7 +3657,7 @@ $sys_id =
$lflags =
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -3721,7 +3721,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -3817,7 +3817,7 @@ $sys_id =
$lflags = -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -4201,7 +4201,7 @@ $sys_id = MINGW32
$lflags = -lws2_32 -lgdi32 -lcrypt32
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT EXPORT_VAR_AS_FN
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -4809,7 +4809,7 @@ $sys_id =
$lflags = -lsocket -lnsl
$bn_ops = DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -4841,7 +4841,7 @@ $sys_id =
$lflags = -lsocket -lnsl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -5097,7 +5097,7 @@ $sys_id =
$lflags = -lsocket -lnsl -ldl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -5545,7 +5545,7 @@ $sys_id =
$lflags = -lsocket -lnsl
$bn_ops = BN_LLONG MD2_CHAR RC4_INDEX DES_PTR DES_RISC1 DES_UNROLL
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -5577,7 +5577,7 @@ $sys_id =
$lflags = -lsocket -lnsl
$bn_ops = BN_LLONG DES_PTR DES_RISC1 DES_UNROLL RC4_INDEX MD2_INT
$cpuid_obj = x86cpuid.o
$bn_obj = bn-586.o co-586.o x86-mont.o
$bn_obj = bn-586.o co-586.o x86-mont.o x86-gf2m.o
$des_obj = des-586.o crypt586.o
$aes_obj = aes-586.o aesni-x86.o
$bf_obj = bf-586.o
@ -5696,6 +5696,38 @@ $ranlib =
$arflags =
$multilib =
*** vxworks-ppc60x
$cc = ccppc
$cflags = -D_REENTRANT -mrtp -mhard-float -mstrict-align -fno-implicit-fp -DPPC32_fp60x -O2 -fstrength-reduce -fno-builtin -fno-strict-aliasing -Wall -DCPU=PPC32 -DTOOL_FAMILY=gnu -DTOOL=gnu -I$(WIND_BASE)/target/usr/h -I$(WIND_BASE)/target/usr/h/wrn/coreip
$unistd =
$thread_cflag =
$sys_id = VXWORKS
$lflags = -Wl,--defsym,__wrs_rtp_base=0xe0000000 -L $(WIND_BASE)/target/usr/lib/ppc/PPC32/common
$bn_ops =
$cpuid_obj =
$bn_obj =
$des_obj =
$aes_obj =
$bf_obj =
$md5_obj =
$sha1_obj =
$cast_obj =
$rc4_obj =
$rmd160_obj =
$rc5_obj =
$wp_obj =
$cmll_obj =
$modes_obj =
$perlasm_scheme =
$dso_scheme =
$shared_target=
$shared_cflag =
$shared_ldflag =
$shared_extension =
$ranlib =
$arflags =
$multilib =
*** vxworks-ppc750
$cc = ccppc
$cflags = -ansi -nostdinc -DPPC750 -D_REENTRANT -fvolatile -fno-builtin -fno-for-scope -fsigned-char -Wall -msoft-float -mlongcall -DCPU=PPC604 -I$(WIND_BASE)/target/h $(DEBUG_FLAG)
@ -5791,3 +5823,35 @@ $shared_extension =
$ranlib =
$arflags =
$multilib =
*** vxworks-ppcgen
$cc = ccppc
$cflags = -D_REENTRANT -mrtp -msoft-float -mstrict-align -O1 -fno-builtin -fno-strict-aliasing -Wall -DCPU=PPC32 -DTOOL_FAMILY=gnu -DTOOL=gnu -I$(WIND_BASE)/target/usr/h -I$(WIND_BASE)/target/usr/h/wrn/coreip
$unistd =
$thread_cflag =
$sys_id = VXWORKS
$lflags = -Wl,--defsym,__wrs_rtp_base=0xe0000000 -L $(WIND_BASE)/target/usr/lib/ppc/PPC32/sfcommon
$bn_ops =
$cpuid_obj =
$bn_obj =
$des_obj =
$aes_obj =
$bf_obj =
$md5_obj =
$sha1_obj =
$cast_obj =
$rc4_obj =
$rmd160_obj =
$rc5_obj =
$wp_obj =
$cmll_obj =
$modes_obj =
$perlasm_scheme =
$dso_scheme =
$shared_target=
$shared_cflag =
$shared_ldflag =
$shared_extension =
$ranlib =
$arflags =
$multilib =

View file

@ -66,6 +66,8 @@ co-586.s: asm/co-586.pl ../perlasm/x86asm.pl
$(PERL) asm/co-586.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@
x86-mont.s: asm/x86-mont.pl ../perlasm/x86asm.pl
$(PERL) asm/x86-mont.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@
x86-gf2m.s: asm/x86-gf2m.pl ../perlasm/x86asm.pl
$(PERL) asm/x86-gf2m.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@
sparcv8.o: asm/sparcv8.S
$(CC) $(CFLAGS) -c asm/sparcv8.S

319
crypto/bn/asm/x86-gf2m.pl Normal file
View file

@ -0,0 +1,319 @@
#!/usr/bin/env perl
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# May 2011
#
# The module implements bn_GF2m_mul_2x2 polynomial multiplication used
# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
# the time being... Except that it has three code paths: pure integer
# code suitable for any x86 CPU, MMX code suitable for PIII and later
# and PCLMULQDQ suitable for Westmere and later. Improvement varies
# from one benchmark and µ-arch to another. Below are interval values
# for 163- and 571-bit ECDH benchmarks relative to compiler-generated
# code:
#
# PIII 16%-30%
# P4 12%-12%
# Opteron 18%-40%
# Core2 19%-44%
# Atom 38%-64%
# Westmere 53%-121%(PCLMULQDQ)/20%-32%(MMX)
# Sandy Bridge 72%-127%(PCLMULQDQ)/27%-23%(MMX)
#
# Note that above improvement coefficients are not coefficients for
# bn_GF2m_mul_2x2 itself. For example 120% ECDH improvement is result
# of bn_GF2m_mul_2x2 being >4x faster. As it gets faster, benchmark
# is more and more dominated by other subroutines, most notably by
# BN_GF2m_mod[_mul]_arr...
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";
&asm_init($ARGV[0],$0,$x86only = $ARGV[$#ARGV] eq "386");
$sse2=0;
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
&external_label("OPENSSL_ia32cap_P") if ($sse2);
$a="eax";
$b="ebx";
($a1,$a2,$a4)=("ecx","edx","ebp");
$R="mm0";
@T=("mm1","mm2");
($A,$B,$B30,$B31)=("mm2","mm3","mm4","mm5");
@i=("esi","edi");
if (!$x86only) {
&function_begin_B("_mul_1x1_mmx");
&sub ("esp",32+4);
&mov ($a1,$a);
&lea ($a2,&DWP(0,$a,$a));
&and ($a1,0x3fffffff);
&lea ($a4,&DWP(0,$a2,$a2));
&mov (&DWP(0*4,"esp"),0);
&and ($a2,0x7fffffff);
&movd ($A,$a);
&movd ($B,$b);
&mov (&DWP(1*4,"esp"),$a1); # a1
&xor ($a1,$a2); # a1^a2
&pxor ($B31,$B31);
&pxor ($B30,$B30);
&mov (&DWP(2*4,"esp"),$a2); # a2
&xor ($a2,$a4); # a2^a4
&mov (&DWP(3*4,"esp"),$a1); # a1^a2
&pcmpgtd($B31,$A); # broadcast 31st bit
&paddd ($A,$A); # $A<<=1
&xor ($a1,$a2); # a1^a4=a1^a2^a2^a4
&mov (&DWP(4*4,"esp"),$a4); # a4
&xor ($a4,$a2); # a2=a4^a2^a4
&pand ($B31,$B);
&pcmpgtd($B30,$A); # broadcast 30th bit
&mov (&DWP(5*4,"esp"),$a1); # a1^a4
&xor ($a4,$a1); # a1^a2^a4
&psllq ($B31,31);
&pand ($B30,$B);
&mov (&DWP(6*4,"esp"),$a2); # a2^a4
&mov (@i[0],0x7);
&mov (&DWP(7*4,"esp"),$a4); # a1^a2^a4
&mov ($a4,@i[0]);
&and (@i[0],$b);
&shr ($b,3);
&mov (@i[1],$a4);
&psllq ($B30,30);
&and (@i[1],$b);
&shr ($b,3);
&movd ($R,&DWP(0,"esp",@i[0],4));
&mov (@i[0],$a4);
&and (@i[0],$b);
&shr ($b,3);
for($n=1;$n<9;$n++) {
&movd (@T[1],&DWP(0,"esp",@i[1],4));
&mov (@i[1],$a4);
&psllq (@T[1],3*$n);
&and (@i[1],$b);
&shr ($b,3);
&pxor ($R,@T[1]);
push(@i,shift(@i)); push(@T,shift(@T));
}
&movd (@T[1],&DWP(0,"esp",@i[1],4));
&pxor ($R,$B30);
&psllq (@T[1],3*$n++);
&pxor ($R,@T[1]);
&movd (@T[0],&DWP(0,"esp",@i[0],4));
&pxor ($R,$B31);
&psllq (@T[0],3*$n);
&add ("esp",32+4);
&pxor ($R,@T[0]);
&ret ();
&function_end_B("_mul_1x1_mmx");
}
($lo,$hi)=("eax","edx");
@T=("ecx","ebp");
&function_begin_B("_mul_1x1_ialu");
&sub ("esp",32+4);
&mov ($a1,$a);
&lea ($a2,&DWP(0,$a,$a));
&lea ($a4,&DWP(0,"",$a,4));
&and ($a1,0x3fffffff);
&lea (@i[1],&DWP(0,$lo,$lo));
&sar ($lo,31); # broadcast 31st bit
&mov (&DWP(0*4,"esp"),0);
&and ($a2,0x7fffffff);
&mov (&DWP(1*4,"esp"),$a1); # a1
&xor ($a1,$a2); # a1^a2
&mov (&DWP(2*4,"esp"),$a2); # a2
&xor ($a2,$a4); # a2^a4
&mov (&DWP(3*4,"esp"),$a1); # a1^a2
&xor ($a1,$a2); # a1^a4=a1^a2^a2^a4
&mov (&DWP(4*4,"esp"),$a4); # a4
&xor ($a4,$a2); # a2=a4^a2^a4
&mov (&DWP(5*4,"esp"),$a1); # a1^a4
&xor ($a4,$a1); # a1^a2^a4
&sar (@i[1],31); # broardcast 30th bit
&and ($lo,$b);
&mov (&DWP(6*4,"esp"),$a2); # a2^a4
&and (@i[1],$b);
&mov (&DWP(7*4,"esp"),$a4); # a1^a2^a4
&mov ($hi,$lo);
&shl ($lo,31);
&mov (@T[0],@i[1]);
&shr ($hi,1);
&mov (@i[0],0x7);
&shl (@i[1],30);
&and (@i[0],$b);
&shr (@T[0],2);
&xor ($lo,@i[1]);
&shr ($b,3);
&mov (@i[1],0x7); # 5-byte instruction!?
&and (@i[1],$b);
&shr ($b,3);
&xor ($hi,@T[0]);
&xor ($lo,&DWP(0,"esp",@i[0],4));
&mov (@i[0],0x7);
&and (@i[0],$b);
&shr ($b,3);
for($n=1;$n<9;$n++) {
&mov (@T[1],&DWP(0,"esp",@i[1],4));
&mov (@i[1],0x7);
&mov (@T[0],@T[1]);
&shl (@T[1],3*$n);
&and (@i[1],$b);
&shr (@T[0],32-3*$n);
&xor ($lo,@T[1]);
&shr ($b,3);
&xor ($hi,@T[0]);
push(@i,shift(@i)); push(@T,shift(@T));
}
&mov (@T[1],&DWP(0,"esp",@i[1],4));
&mov (@T[0],@T[1]);
&shl (@T[1],3*$n);
&mov (@i[1],&DWP(0,"esp",@i[0],4));
&shr (@T[0],32-3*$n); $n++;
&mov (@i[0],@i[1]);
&xor ($lo,@T[1]);
&shl (@i[1],3*$n);
&xor ($hi,@T[0]);
&shr (@i[0],32-3*$n);
&xor ($lo,@i[1]);
&xor ($hi,@i[0]);
&add ("esp",32+4);
&ret ();
&function_end_B("_mul_1x1_ialu");
sub pclmulqdq
{ my($dst,$src,$imm)=@_;
if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
{ &data_byte(0x66,0x0f,0x3a,0x44,0xc0|($1<<3)|$2,$imm); }
}
# void bn_GF2m_mul_2x2(BN_ULONG *r, BN_ULONG a1, BN_ULONG a0, BN_ULONG b1, BN_ULONG b0);
&function_begin_B("bn_GF2m_mul_2x2");
if (!$x86only) {
&picmeup("edx","OPENSSL_ia32cap_P");
&mov ("eax",&DWP(0,"edx"));
&mov ("edx",&DWP(4,"edx"));
&test ("eax",1<<23); # check MMX bit
&jz (&label("ialu"));
if ($sse2) {
&test ("eax",1<<24); # check FXSR bit
&jz (&label("mmx"));
&test ("edx",1<<1); # check PCLMULQDQ bit
&jz (&label("mmx"));
&movups ("xmm0",&QWP(8,"esp"));
&shufps ("xmm0","xmm0",0b10110001);
&pclmulqdq ("xmm0","xmm0",1);
&mov ("eax",&DWP(4,"esp"));
&movups (&QWP(0,"eax"),"xmm0");
&ret ();
&set_label("mmx",16);
}
&push ("ebp");
&push ("ebx");
&push ("esi");
&push ("edi");
&mov ($a,&wparam(1));
&mov ($b,&wparam(3));
&call ("_mul_1x1_mmx"); # a1·b1
&movq ("mm7",$R);
&mov ($a,&wparam(2));
&mov ($b,&wparam(4));
&call ("_mul_1x1_mmx"); # a0·b0
&movq ("mm6",$R);
&mov ($a,&wparam(1));
&mov ($b,&wparam(3));
&xor ($a,&wparam(2));
&xor ($b,&wparam(4));
&call ("_mul_1x1_mmx"); # (a0+a1)·(b0+b1)
&pxor ($R,"mm7");
&mov ($a,&wparam(0));
&pxor ($R,"mm6"); # (a0+a1)·(b0+b1)-a1·b1-a0·b0
&movq ($A,$R);
&psllq ($R,32);
&pop ("edi");
&psrlq ($A,32);
&pop ("esi");
&pxor ($R,"mm6");
&pop ("ebx");
&pxor ($A,"mm7");
&movq (&QWP(0,$a),$R);
&pop ("ebp");
&movq (&QWP(8,$a),$A);
&emms ();
&ret ();
&set_label("ialu",16);
}
&push ("ebp");
&push ("ebx");
&push ("esi");
&push ("edi");
&stack_push(4+1);
&mov ($a,&wparam(1));
&mov ($b,&wparam(3));
&call ("_mul_1x1_ialu"); # a1·b1
&mov (&DWP(8,"esp"),$lo);
&mov (&DWP(12,"esp"),$hi);
&mov ($a,&wparam(2));
&mov ($b,&wparam(4));
&call ("_mul_1x1_ialu"); # a0·b0
&mov (&DWP(0,"esp"),$lo);
&mov (&DWP(4,"esp"),$hi);
&mov ($a,&wparam(1));
&mov ($b,&wparam(3));
&xor ($a,&wparam(2));
&xor ($b,&wparam(4));
&call ("_mul_1x1_ialu"); # (a0+a1)·(b0+b1)
&mov ("ebp",&wparam(0));
@r=("ebx","ecx","edi","esi");
&mov (@r[0],&DWP(0,"esp"));
&mov (@r[1],&DWP(4,"esp"));
&mov (@r[2],&DWP(8,"esp"));
&mov (@r[3],&DWP(12,"esp"));
&xor ($lo,$hi);
&xor ($hi,@r[1]);
&xor ($lo,@r[0]);
&mov (&DWP(0,"ebp"),@r[0]);
&xor ($hi,@r[2]);
&mov (&DWP(12,"ebp"),@r[3]);
&xor ($lo,@r[3]);
&stack_pop(4+1);
&xor ($hi,@r[3]);
&pop ("edi");
&xor ($lo,$hi);
&pop ("esi");
&mov (&DWP(8,"ebp"),$hi);
&pop ("ebx");
&mov (&DWP(4,"ebp"),$lo);
&pop ("ebp");
&ret ();
&function_end_B("bn_GF2m_mul_2x2");
&asciz ("GF2m Multiplication for x86, CRYPTOGAMS by <appro\@openssl.org>");
&asm_finish();