wbrawner/openssl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
openssl/crypto/ec/asm/ecp_nistz256-x86.pl

1867 lines
56 KiB
Perl
Raw Normal View History

Add OpenSSL copyright to .pl files Reviewed-by: Richard Levitte <levitte@openssl.org>
2016-05-21 12:23:39 +00:00
#! /usr/bin/env perl
Update copyright year Reviewed-by: Richard Levitte <levitte@openssl.org> (Merged from https://github.com/openssl/openssl/pull/7176)
2018-09-11 12:22:14 +00:00
# Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved.
Add OpenSSL copyright to .pl files Reviewed-by: Richard Levitte <levitte@openssl.org>
2016-05-21 12:23:39 +00:00
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# ECP_NISTZ256 module for x86/SSE2.
#
# October 2014.
#
# Original ECP_NISTZ256 submission targeting x86_64 is detailed in
# http://eprint.iacr.org/2013/816. In the process of adaptation
# original .c module was made 32-bit savvy in order to make this
# implementation possible.
#
# with/without -DECP_NISTZ256_ASM
# Pentium +66-163%
# PIII +72-172%
# P4 +65-132%
# Core2 +90-215%
# Sandy Bridge +105-265% (contemporary i[57]-* are all close to this)
# Atom +65-155%
# Opteron +54-110%
# Bulldozer +99-240%
# VIA Nano +93-290%
#
# Ranges denote minimum and maximum improvement coefficients depending
# on benchmark. Lower coefficients are for ECDSA sign, server-side
# operation. Keep in mind that +200% means 3x improvement.
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";
Unified - adapt the generation of ec assembler to use GENERATE This gets rid of the BEGINRAW..ENDRAW sections in crypto/ec/build.info. This also moves the assembler generating perl scripts to take the output file name as last command line argument, where necessary. Reviewed-by: Andy Polyakov <appro@openssl.org>
2016-03-07 14:47:09 +00:00
$output=pop;
open STDOUT,">$output";
Remove filename argument to x86 asm_init. The assembler already knows the actual path to the generated file and, in other perlasm architectures, is left to manage debug symbols itself. Notably, in OpenSSL 1.1.x's new build system, which allows a separate build directory, converting .pl to .s as the scripts currently do result in the wrong paths. This also avoids inconsistencies from some of the files using $0 and some passing in the filename. Reviewed-by: Richard Levitte <levitte@openssl.org> Reviewed-by: Andy Polyakov <appro@openssl.org> (Merged from https://github.com/openssl/openssl/pull/3431)
2017-05-10 18:24:56 +00:00
&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
$sse2=0;
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
&external_label("OPENSSL_ia32cap_P") if ($sse2);
########################################################################
# Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
#
open TABLE,"<ecp_nistz256_table.c" or
open TABLE,"<${dir}../ecp_nistz256_table.c" or
die "failed to open ecp_nistz256_table.c:",$!;
use integer;
foreach(<TABLE>) {
s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
}
close TABLE;
# See ecp_nistz256_table.c for explanation for why it's 64*16*37.
# 64*16*37-1 is because $#arr returns last valid index or @arr, not
# amount of elements.
die "insane number of elements" if ($#arr != 64*16*37-1);
&public_label("ecp_nistz256_precomputed");
&align(4096);
&set_label("ecp_nistz256_precomputed");
########################################################################
# this conversion smashes P256_POINT_AFFINE by individual bytes with
# 64 byte interval, similar to
# 1111222233334444
# 1234123412341234
for(1..37) {
@tbl = splice(@arr,0,64*16);
for($i=0;$i<64;$i++) {
undef @line;
for($j=0;$j<64;$j++) {
push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
}
&data_byte(join(',',map { sprintf "0x%02x",$_} @line));
}
}
########################################################################
# Keep in mind that constants are stored least to most significant word
&static_label("RR");
&set_label("RR",64);
&data_word(3,0,-1,-5,-2,-1,-3,4); # 2^512 mod P-256
&static_label("ONE_mont");
&set_label("ONE_mont");
&data_word(1,0,0,-1,-1,-1,-2,0);
&static_label("ONE");
&set_label("ONE");
&data_word(1,0,0,0,0,0,0,0);
&asciz("ECP_NISZ256 for x86/SSE2, CRYPTOGAMS by <appro\@openssl.org>");
&align(64);
########################################################################
# void ecp_nistz256_mul_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_mul_by_2");
&mov ("esi",&wparam(1));
&mov ("edi",&wparam(0));
&mov ("ebp","esi");
########################################################################
# common pattern for internal functions is that %edi is result pointer,
# %esi and %ebp are input ones, %ebp being optional. %edi is preserved.
&call ("_ecp_nistz256_add");
&function_end("ecp_nistz256_mul_by_2");
########################################################################
# void ecp_nistz256_mul_by_3(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_mul_by_3");
&mov ("esi",&wparam(1));
# multiplication by 3 is performed
# as 2*n+n, but we can't use output
# to store 2*n, because if output
# pointer equals to input, then
# we'll get 2*n+2*n.
&stack_push(8); # therefore we need to allocate
# 256-bit intermediate buffer.
&mov ("edi","esp");
&mov ("ebp","esi");
&call ("_ecp_nistz256_add");
&lea ("esi",&DWP(0,"edi"));
&mov ("ebp",&wparam(1));
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_add");
&stack_pop(8);
&function_end("ecp_nistz256_mul_by_3");
########################################################################
# void ecp_nistz256_div_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_div_by_2");
&mov ("esi",&wparam(1));
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_div_by_2");
&function_end("ecp_nistz256_div_by_2");
&function_begin_B("_ecp_nistz256_div_by_2");
# tmp = a is odd ? a+mod : a
#
# note that because mod has special form, i.e. consists of
# 0xffffffff, 1 and 0s, we can conditionally synthesize it by
# assigning least significant bit of input to one register,
# %ebp, and its negative to another, %edx.
&mov ("ebp",&DWP(0,"esi"));
&xor ("edx","edx");
&mov ("ebx",&DWP(4,"esi"));
&mov ("eax","ebp");
&and ("ebp",1);
&mov ("ecx",&DWP(8,"esi"));
&sub ("edx","ebp");
&add ("eax","edx");
&adc ("ebx","edx");
&mov (&DWP(0,"edi"),"eax");
&adc ("ecx","edx");
&mov (&DWP(4,"edi"),"ebx");
&mov (&DWP(8,"edi"),"ecx");
&mov ("eax",&DWP(12,"esi"));
&mov ("ebx",&DWP(16,"esi"));
&adc ("eax",0);
&mov ("ecx",&DWP(20,"esi"));
&adc ("ebx",0);
&mov (&DWP(12,"edi"),"eax");
&adc ("ecx",0);
&mov (&DWP(16,"edi"),"ebx");
&mov (&DWP(20,"edi"),"ecx");
&mov ("eax",&DWP(24,"esi"));
&mov ("ebx",&DWP(28,"esi"));
&adc ("eax","ebp");
&adc ("ebx","edx");
&mov (&DWP(24,"edi"),"eax");
&sbb ("esi","esi"); # broadcast carry bit
&mov (&DWP(28,"edi"),"ebx");
# ret = tmp >> 1
&mov ("eax",&DWP(0,"edi"));
&mov ("ebx",&DWP(4,"edi"));
&mov ("ecx",&DWP(8,"edi"));
&mov ("edx",&DWP(12,"edi"));
&shr ("eax",1);
&mov ("ebp","ebx");
&shl ("ebx",31);
&or ("eax","ebx");
&shr ("ebp",1);
&mov ("ebx","ecx");
&shl ("ecx",31);
&mov (&DWP(0,"edi"),"eax");
&or ("ebp","ecx");
&mov ("eax",&DWP(16,"edi"));
&shr ("ebx",1);
&mov ("ecx","edx");
&shl ("edx",31);
&mov (&DWP(4,"edi"),"ebp");
&or ("ebx","edx");
&mov ("ebp",&DWP(20,"edi"));
&shr ("ecx",1);
&mov ("edx","eax");
&shl ("eax",31);
&mov (&DWP(8,"edi"),"ebx");
&or ("ecx","eax");
&mov ("ebx",&DWP(24,"edi"));
&shr ("edx",1);
&mov ("eax","ebp");
&shl ("ebp",31);
&mov (&DWP(12,"edi"),"ecx");
&or ("edx","ebp");
&mov ("ecx",&DWP(28,"edi"));
&shr ("eax",1);
&mov ("ebp","ebx");
&shl ("ebx",31);
&mov (&DWP(16,"edi"),"edx");
&or ("eax","ebx");
&shr ("ebp",1);
&mov ("ebx","ecx");
&shl ("ecx",31);
&mov (&DWP(20,"edi"),"eax");
&or ("ebp","ecx");
&shr ("ebx",1);
&shl ("esi",31);
&mov (&DWP(24,"edi"),"ebp");
&or ("ebx","esi"); # handle top-most carry bit
&mov (&DWP(28,"edi"),"ebx");
&ret ();
&function_end_B("_ecp_nistz256_div_by_2");
########################################################################
# void ecp_nistz256_add(BN_ULONG edi[8],const BN_ULONG esi[8],
# const BN_ULONG ebp[8]);
&function_begin("ecp_nistz256_add");
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_add");
&function_end("ecp_nistz256_add");
&function_begin_B("_ecp_nistz256_add");
&mov ("eax",&DWP(0,"esi"));
&mov ("ebx",&DWP(4,"esi"));
&mov ("ecx",&DWP(8,"esi"));
&add ("eax",&DWP(0,"ebp"));
&mov ("edx",&DWP(12,"esi"));
&adc ("ebx",&DWP(4,"ebp"));
&mov (&DWP(0,"edi"),"eax");
&adc ("ecx",&DWP(8,"ebp"));
&mov (&DWP(4,"edi"),"ebx");
&adc ("edx",&DWP(12,"ebp"));
&mov (&DWP(8,"edi"),"ecx");
&mov (&DWP(12,"edi"),"edx");
&mov ("eax",&DWP(16,"esi"));
&mov ("ebx",&DWP(20,"esi"));
&mov ("ecx",&DWP(24,"esi"));
&adc ("eax",&DWP(16,"ebp"));
&mov ("edx",&DWP(28,"esi"));
&adc ("ebx",&DWP(20,"ebp"));
&mov (&DWP(16,"edi"),"eax");
&adc ("ecx",&DWP(24,"ebp"));
&mov (&DWP(20,"edi"),"ebx");
ec/asm/ecp_nistz256-*.pl: addition to perform stricter reduction. Addition was not preserving inputs' property of being fully reduced. Thanks to Brian Smith for reporting this. Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-20 20:10:24 +00:00
&mov ("esi",0);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&adc ("edx",&DWP(28,"ebp"));
&mov (&DWP(24,"edi"),"ecx");
ec/asm/ecp_nistz256-*.pl: addition to perform stricter reduction. Addition was not preserving inputs' property of being fully reduced. Thanks to Brian Smith for reporting this. Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-20 20:10:24 +00:00
&adc ("esi",0);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP(28,"edi"),"edx");
ec/asm/ecp_nistz256-*.pl: addition to perform stricter reduction. Addition was not preserving inputs' property of being fully reduced. Thanks to Brian Smith for reporting this. Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-20 20:10:24 +00:00
# if a+b >= modulus, subtract modulus.
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
#
ec/asm/ecp_nistz256-*.pl: addition to perform stricter reduction. Addition was not preserving inputs' property of being fully reduced. Thanks to Brian Smith for reporting this. Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-20 20:10:24 +00:00
# But since comparison implies subtraction, we subtract modulus
# to see if it borrows, and then subtract it for real if
# subtraction didn't borrow.
&mov ("eax",&DWP(0,"edi"));
&mov ("ebx",&DWP(4,"edi"));
&mov ("ecx",&DWP(8,"edi"));
&sub ("eax",-1);
&mov ("edx",&DWP(12,"edi"));
&sbb ("ebx",-1);
&mov ("eax",&DWP(16,"edi"));
&sbb ("ecx",-1);
&mov ("ebx",&DWP(20,"edi"));
&sbb ("edx",0);
&mov ("ecx",&DWP(24,"edi"));
&sbb ("eax",0);
&mov ("edx",&DWP(28,"edi"));
&sbb ("ebx",0);
&sbb ("ecx",1);
&sbb ("edx",-1);
&sbb ("esi",0);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
# Note that because mod has special form, i.e. consists of
# 0xffffffff, 1 and 0s, we can conditionally synthesize it by
ec/asm/ecp_nistz256-*.pl: addition to perform stricter reduction. Addition was not preserving inputs' property of being fully reduced. Thanks to Brian Smith for reporting this. Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-20 20:10:24 +00:00
# by using borrow.
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
ec/asm/ecp_nistz256-*.pl: addition to perform stricter reduction. Addition was not preserving inputs' property of being fully reduced. Thanks to Brian Smith for reporting this. Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-20 20:10:24 +00:00
&not ("esi");
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov ("eax",&DWP(0,"edi"));
&mov ("ebp","esi");
&mov ("ebx",&DWP(4,"edi"));
&shr ("ebp",31);
&mov ("ecx",&DWP(8,"edi"));
&sub ("eax","esi");
&mov ("edx",&DWP(12,"edi"));
&sbb ("ebx","esi");
&mov (&DWP(0,"edi"),"eax");
&sbb ("ecx","esi");
&mov (&DWP(4,"edi"),"ebx");
&sbb ("edx",0);
&mov (&DWP(8,"edi"),"ecx");
&mov (&DWP(12,"edi"),"edx");
&mov ("eax",&DWP(16,"edi"));
&mov ("ebx",&DWP(20,"edi"));
&mov ("ecx",&DWP(24,"edi"));
&sbb ("eax",0);
&mov ("edx",&DWP(28,"edi"));
&sbb ("ebx",0);
&mov (&DWP(16,"edi"),"eax");
&sbb ("ecx","ebp");
&mov (&DWP(20,"edi"),"ebx");
&sbb ("edx","esi");
&mov (&DWP(24,"edi"),"ecx");
&mov (&DWP(28,"edi"),"edx");
&ret ();
&function_end_B("_ecp_nistz256_add");
########################################################################
# void ecp_nistz256_sub(BN_ULONG edi[8],const BN_ULONG esi[8],
# const BN_ULONG ebp[8]);
&function_begin("ecp_nistz256_sub");
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_sub");
&function_end("ecp_nistz256_sub");
&function_begin_B("_ecp_nistz256_sub");
&mov ("eax",&DWP(0,"esi"));
&mov ("ebx",&DWP(4,"esi"));
&mov ("ecx",&DWP(8,"esi"));
&sub ("eax",&DWP(0,"ebp"));
&mov ("edx",&DWP(12,"esi"));
&sbb ("ebx",&DWP(4,"ebp"));
&mov (&DWP(0,"edi"),"eax");
&sbb ("ecx",&DWP(8,"ebp"));
&mov (&DWP(4,"edi"),"ebx");
&sbb ("edx",&DWP(12,"ebp"));
&mov (&DWP(8,"edi"),"ecx");
&mov (&DWP(12,"edi"),"edx");
&mov ("eax",&DWP(16,"esi"));
&mov ("ebx",&DWP(20,"esi"));
&mov ("ecx",&DWP(24,"esi"));
&sbb ("eax",&DWP(16,"ebp"));
&mov ("edx",&DWP(28,"esi"));
&sbb ("ebx",&DWP(20,"ebp"));
&sbb ("ecx",&DWP(24,"ebp"));
&mov (&DWP(16,"edi"),"eax");
&sbb ("edx",&DWP(28,"ebp"));
&mov (&DWP(20,"edi"),"ebx");
&sbb ("esi","esi"); # broadcast borrow bit
&mov (&DWP(24,"edi"),"ecx");
&mov (&DWP(28,"edi"),"edx");
# if a-b borrows, add modulus.
#
# Note that because mod has special form, i.e. consists of
# 0xffffffff, 1 and 0s, we can conditionally synthesize it by
# assigning borrow bit to one register, %ebp, and its negative
# to another, %esi. But we started by calculating %esi...
&mov ("eax",&DWP(0,"edi"));
&mov ("ebp","esi");
&mov ("ebx",&DWP(4,"edi"));
&shr ("ebp",31);
&mov ("ecx",&DWP(8,"edi"));
&add ("eax","esi");
&mov ("edx",&DWP(12,"edi"));
&adc ("ebx","esi");
&mov (&DWP(0,"edi"),"eax");
&adc ("ecx","esi");
&mov (&DWP(4,"edi"),"ebx");
&adc ("edx",0);
&mov (&DWP(8,"edi"),"ecx");
&mov (&DWP(12,"edi"),"edx");
&mov ("eax",&DWP(16,"edi"));
&mov ("ebx",&DWP(20,"edi"));
&mov ("ecx",&DWP(24,"edi"));
&adc ("eax",0);
&mov ("edx",&DWP(28,"edi"));
&adc ("ebx",0);
&mov (&DWP(16,"edi"),"eax");
&adc ("ecx","ebp");
&mov (&DWP(20,"edi"),"ebx");
&adc ("edx","esi");
&mov (&DWP(24,"edi"),"ecx");
&mov (&DWP(28,"edi"),"edx");
&ret ();
&function_end_B("_ecp_nistz256_sub");
########################################################################
# void ecp_nistz256_neg(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_neg");
&mov ("ebp",&wparam(1));
&mov ("edi",&wparam(0));
&xor ("eax","eax");
&stack_push(8);
&mov (&DWP(0,"esp"),"eax");
&mov ("esi","esp");
&mov (&DWP(4,"esp"),"eax");
&mov (&DWP(8,"esp"),"eax");
&mov (&DWP(12,"esp"),"eax");
&mov (&DWP(16,"esp"),"eax");
&mov (&DWP(20,"esp"),"eax");
&mov (&DWP(24,"esp"),"eax");
&mov (&DWP(28,"esp"),"eax");
Remove trailing whitespace from some files. The prevailing style seems to not have trailing whitespace, but a few lines do. This is mostly in the perlasm files, but a few C files got them after the reformat. This is the result of: find . -name '*.pl' | xargs sed -E -i '' -e 's/( |'$'\t'')*$//' find . -name '*.c' | xargs sed -E -i '' -e 's/( |'$'\t'')*$//' find . -name '*.h' | xargs sed -E -i '' -e 's/( |'$'\t'')*$//' Then bn_prime.h was excluded since this is a generated file. Note mkerr.pl has some changes in a heredoc for some help output, but other lines there lack trailing whitespace too. Reviewed-by: Kurt Roeckx <kurt@openssl.org> Reviewed-by: Matt Caswell <matt@openssl.org>
2016-10-10 16:01:24 +00:00
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&call ("_ecp_nistz256_sub");
&stack_pop(8);
&function_end("ecp_nistz256_neg");
&function_begin_B("_picup_eax");
&mov ("eax",&DWP(0,"esp"));
&ret ();
&function_end_B("_picup_eax");
########################################################################
# void ecp_nistz256_to_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_to_mont");
&mov ("esi",&wparam(1));
&call ("_picup_eax");
&set_label("pic");
&lea ("ebp",&DWP(&label("RR")."-".&label("pic"),"eax"));
if ($sse2) {
&picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("eax",&DWP(0,"eax")); }
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_mul_mont");
&function_end("ecp_nistz256_to_mont");
########################################################################
# void ecp_nistz256_from_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_from_mont");
&mov ("esi",&wparam(1));
&call ("_picup_eax");
&set_label("pic");
&lea ("ebp",&DWP(&label("ONE")."-".&label("pic"),"eax"));
if ($sse2) {
&picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("eax",&DWP(0,"eax")); }
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_mul_mont");
&function_end("ecp_nistz256_from_mont");
########################################################################
# void ecp_nistz256_mul_mont(BN_ULONG edi[8],const BN_ULONG esi[8],
# const BN_ULONG ebp[8]);
&function_begin("ecp_nistz256_mul_mont");
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
if ($sse2) {
&call ("_picup_eax");
&set_label("pic");
&picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("eax",&DWP(0,"eax")); }
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_mul_mont");
&function_end("ecp_nistz256_mul_mont");
########################################################################
# void ecp_nistz256_sqr_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
&function_begin("ecp_nistz256_sqr_mont");
&mov ("esi",&wparam(1));
if ($sse2) {
&call ("_picup_eax");
&set_label("pic");
&picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("eax",&DWP(0,"eax")); }
&mov ("edi",&wparam(0));
&mov ("ebp","esi");
&call ("_ecp_nistz256_mul_mont");
&function_end("ecp_nistz256_sqr_mont");
&function_begin_B("_ecp_nistz256_mul_mont");
if ($sse2) {
&and ("eax",1<<24|1<<26);
&cmp ("eax",1<<24|1<<26); # see if XMM+SSE2 is on
&jne (&label("mul_mont_ialu"));
########################################
# SSE2 code path featuring 32x16-bit
# multiplications is ~2x faster than
# IALU counterpart (except on Atom)...
########################################
# stack layout:
# +------------------------------------+< %esp
# | 7 16-byte temporary XMM words, |
# | "sliding" toward lower address |
# . .
# +------------------------------------+
# | unused XMM word |
# +------------------------------------+< +128,%ebx
# | 8 16-byte XMM words holding copies |
# | of a[i]<<64|a[i] |
# . .
# . .
# +------------------------------------+< +256
&mov ("edx","esp");
&sub ("esp",0x100);
&movd ("xmm7",&DWP(0,"ebp")); # b[0] -> 0000.00xy
&lea ("ebp",&DWP(4,"ebp"));
&pcmpeqd("xmm6","xmm6");
&psrlq ("xmm6",48); # compose 0xffff<<64|0xffff
&pshuflw("xmm7","xmm7",0b11011100); # 0000.00xy -> 0000.0x0y
&and ("esp",-64);
&pshufd ("xmm7","xmm7",0b11011100); # 0000.0x0y -> 000x.000y
&lea ("ebx",&DWP(0x80,"esp"));
&movd ("xmm0",&DWP(4*0,"esi")); # a[0] -> 0000.00xy
&pshufd ("xmm0","xmm0",0b11001100); # 0000.00xy -> 00xy.00xy
&movd ("xmm1",&DWP(4*1,"esi")); # a[1] -> ...
&movdqa (&QWP(0x00,"ebx"),"xmm0"); # offload converted a[0]
&pmuludq("xmm0","xmm7"); # a[0]*b[0]
&movd ("xmm2",&DWP(4*2,"esi"));
&pshufd ("xmm1","xmm1",0b11001100);
&movdqa (&QWP(0x10,"ebx"),"xmm1");
&pmuludq("xmm1","xmm7"); # a[1]*b[0]
&movq ("xmm4","xmm0"); # clear upper 64 bits
&pslldq("xmm4",6);
&paddq ("xmm4","xmm0");
&movdqa("xmm5","xmm4");
&psrldq("xmm4",10); # upper 32 bits of a[0]*b[0]
&pand ("xmm5","xmm6"); # lower 32 bits of a[0]*b[0]
# Upper half of a[0]*b[i] is carried into next multiplication
# iteration, while lower one "participates" in actual reduction.
# Normally latter is done by accumulating result of multiplication
# of modulus by "magic" digit, but thanks to special form of modulus
# and "magic" digit it can be performed only with additions and
# subtractions (see note in IALU section below). Note that we are
# not bothered with carry bits, they are accumulated in "flatten"
# phase after all multiplications and reductions.
&movd ("xmm3",&DWP(4*3,"esi"));
&pshufd ("xmm2","xmm2",0b11001100);
&movdqa (&QWP(0x20,"ebx"),"xmm2");
&pmuludq("xmm2","xmm7"); # a[2]*b[0]
&paddq ("xmm1","xmm4"); # a[1]*b[0]+hw(a[0]*b[0]), carry
&movdqa (&QWP(0x00,"esp"),"xmm1"); # t[0]
&movd ("xmm0",&DWP(4*4,"esi"));
&pshufd ("xmm3","xmm3",0b11001100);
&movdqa (&QWP(0x30,"ebx"),"xmm3");
&pmuludq("xmm3","xmm7"); # a[3]*b[0]
&movdqa (&QWP(0x10,"esp"),"xmm2");
&movd ("xmm1",&DWP(4*5,"esi"));
&pshufd ("xmm0","xmm0",0b11001100);
&movdqa (&QWP(0x40,"ebx"),"xmm0");
&pmuludq("xmm0","xmm7"); # a[4]*b[0]
&paddq ("xmm3","xmm5"); # a[3]*b[0]+lw(a[0]*b[0]), reduction step
&movdqa (&QWP(0x20,"esp"),"xmm3");
&movd ("xmm2",&DWP(4*6,"esi"));
&pshufd ("xmm1","xmm1",0b11001100);
&movdqa (&QWP(0x50,"ebx"),"xmm1");
&pmuludq("xmm1","xmm7"); # a[5]*b[0]
&movdqa (&QWP(0x30,"esp"),"xmm0");
&pshufd("xmm4","xmm5",0b10110001); # xmm4 = xmm5<<32, reduction step
&movd ("xmm3",&DWP(4*7,"esi"));
&pshufd ("xmm2","xmm2",0b11001100);
&movdqa (&QWP(0x60,"ebx"),"xmm2");
&pmuludq("xmm2","xmm7"); # a[6]*b[0]
&movdqa (&QWP(0x40,"esp"),"xmm1");
&psubq ("xmm4","xmm5"); # xmm4 = xmm5*0xffffffff, reduction step
&movd ("xmm0",&DWP(0,"ebp")); # b[1] -> 0000.00xy
&pshufd ("xmm3","xmm3",0b11001100);
&movdqa (&QWP(0x70,"ebx"),"xmm3");
&pmuludq("xmm3","xmm7"); # a[7]*b[0]
&pshuflw("xmm7","xmm0",0b11011100); # 0000.00xy -> 0000.0x0y
&movdqa ("xmm0",&QWP(0x00,"ebx")); # pre-load converted a[0]
&pshufd ("xmm7","xmm7",0b11011100); # 0000.0x0y -> 000x.000y
&mov ("ecx",6);
&lea ("ebp",&DWP(4,"ebp"));
&jmp (&label("madd_sse2"));
&set_label("madd_sse2",16);
&paddq ("xmm2","xmm5"); # a[6]*b[i-1]+lw(a[0]*b[i-1]), reduction step [modulo-scheduled]
&paddq ("xmm3","xmm4"); # a[7]*b[i-1]+lw(a[0]*b[i-1])*0xffffffff, reduction step [modulo-scheduled]
&movdqa ("xmm1",&QWP(0x10,"ebx"));
&pmuludq("xmm0","xmm7"); # a[0]*b[i]
&movdqa(&QWP(0x50,"esp"),"xmm2");
&movdqa ("xmm2",&QWP(0x20,"ebx"));
&pmuludq("xmm1","xmm7"); # a[1]*b[i]
&movdqa(&QWP(0x60,"esp"),"xmm3");
&paddq ("xmm0",&QWP(0x00,"esp"));
&movdqa ("xmm3",&QWP(0x30,"ebx"));
&pmuludq("xmm2","xmm7"); # a[2]*b[i]
&movq ("xmm4","xmm0"); # clear upper 64 bits
&pslldq("xmm4",6);
&paddq ("xmm1",&QWP(0x10,"esp"));
&paddq ("xmm4","xmm0");
&movdqa("xmm5","xmm4");
&psrldq("xmm4",10); # upper 33 bits of a[0]*b[i]+t[0]
&movdqa ("xmm0",&QWP(0x40,"ebx"));
&pmuludq("xmm3","xmm7"); # a[3]*b[i]
&paddq ("xmm1","xmm4"); # a[1]*b[i]+hw(a[0]*b[i]), carry
&paddq ("xmm2",&QWP(0x20,"esp"));
&movdqa (&QWP(0x00,"esp"),"xmm1");
&movdqa ("xmm1",&QWP(0x50,"ebx"));
&pmuludq("xmm0","xmm7"); # a[4]*b[i]
&paddq ("xmm3",&QWP(0x30,"esp"));
&movdqa (&QWP(0x10,"esp"),"xmm2");
&pand ("xmm5","xmm6"); # lower 32 bits of a[0]*b[i]
ec/asm/ecp_nistz256-x86.pl: fix typos (error shows in Windows build). Reviewed-by: Matt Caswell <matt@openssl.org>
2015-02-10 20:52:25 +00:00
&movdqa ("xmm2",&QWP(0x60,"ebx"));
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&pmuludq("xmm1","xmm7"); # a[5]*b[i]
&paddq ("xmm3","xmm5"); # a[3]*b[i]+lw(a[0]*b[i]), reduction step
&paddq ("xmm0",&QWP(0x40,"esp"));
&movdqa (&QWP(0x20,"esp"),"xmm3");
&pshufd("xmm4","xmm5",0b10110001); # xmm4 = xmm5<<32, reduction step
&movdqa ("xmm3","xmm7");
&pmuludq("xmm2","xmm7"); # a[6]*b[i]
&movd ("xmm7",&DWP(0,"ebp")); # b[i++] -> 0000.00xy
&lea ("ebp",&DWP(4,"ebp"));
&paddq ("xmm1",&QWP(0x50,"esp"));
&psubq ("xmm4","xmm5"); # xmm4 = xmm5*0xffffffff, reduction step
&movdqa (&QWP(0x30,"esp"),"xmm0");
&pshuflw("xmm7","xmm7",0b11011100); # 0000.00xy -> 0000.0x0y
&pmuludq("xmm3",&QWP(0x70,"ebx")); # a[7]*b[i]
&pshufd("xmm7","xmm7",0b11011100); # 0000.0x0y -> 000x.000y
&movdqa("xmm0",&QWP(0x00,"ebx")); # pre-load converted a[0]
&movdqa (&QWP(0x40,"esp"),"xmm1");
&paddq ("xmm2",&QWP(0x60,"esp"));
&dec ("ecx");
&jnz (&label("madd_sse2"));
&paddq ("xmm2","xmm5"); # a[6]*b[6]+lw(a[0]*b[6]), reduction step [modulo-scheduled]
&paddq ("xmm3","xmm4"); # a[7]*b[6]+lw(a[0]*b[6])*0xffffffff, reduction step [modulo-scheduled]
&movdqa ("xmm1",&QWP(0x10,"ebx"));
&pmuludq("xmm0","xmm7"); # a[0]*b[7]
&movdqa(&QWP(0x50,"esp"),"xmm2");
&movdqa ("xmm2",&QWP(0x20,"ebx"));
&pmuludq("xmm1","xmm7"); # a[1]*b[7]
&movdqa(&QWP(0x60,"esp"),"xmm3");
&paddq ("xmm0",&QWP(0x00,"esp"));
&movdqa ("xmm3",&QWP(0x30,"ebx"));
&pmuludq("xmm2","xmm7"); # a[2]*b[7]
&movq ("xmm4","xmm0"); # clear upper 64 bits
&pslldq("xmm4",6);
&paddq ("xmm1",&QWP(0x10,"esp"));
&paddq ("xmm4","xmm0");
&movdqa("xmm5","xmm4");
&psrldq("xmm4",10); # upper 33 bits of a[0]*b[i]+t[0]
&movdqa ("xmm0",&QWP(0x40,"ebx"));
&pmuludq("xmm3","xmm7"); # a[3]*b[7]
&paddq ("xmm1","xmm4"); # a[1]*b[7]+hw(a[0]*b[7]), carry
&paddq ("xmm2",&QWP(0x20,"esp"));
&movdqa (&QWP(0x00,"esp"),"xmm1");
&movdqa ("xmm1",&QWP(0x50,"ebx"));
&pmuludq("xmm0","xmm7"); # a[4]*b[7]
&paddq ("xmm3",&QWP(0x30,"esp"));
&movdqa (&QWP(0x10,"esp"),"xmm2");
&pand ("xmm5","xmm6"); # lower 32 bits of a[0]*b[i]
ec/asm/ecp_nistz256-x86.pl: fix typos (error shows in Windows build). Reviewed-by: Matt Caswell <matt@openssl.org>
2015-02-10 20:52:25 +00:00
&movdqa ("xmm2",&QWP(0x60,"ebx"));
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&pmuludq("xmm1","xmm7"); # a[5]*b[7]
&paddq ("xmm3","xmm5"); # reduction step
&paddq ("xmm0",&QWP(0x40,"esp"));
&movdqa (&QWP(0x20,"esp"),"xmm3");
&pshufd("xmm4","xmm5",0b10110001); # xmm4 = xmm5<<32, reduction step
&movdqa ("xmm3",&QWP(0x70,"ebx"));
&pmuludq("xmm2","xmm7"); # a[6]*b[7]
&paddq ("xmm1",&QWP(0x50,"esp"));
&psubq ("xmm4","xmm5"); # xmm4 = xmm5*0xffffffff, reduction step
&movdqa (&QWP(0x30,"esp"),"xmm0");
&pmuludq("xmm3","xmm7"); # a[7]*b[7]
&pcmpeqd("xmm7","xmm7");
&movdqa ("xmm0",&QWP(0x00,"esp"));
&pslldq ("xmm7",8);
&movdqa (&QWP(0x40,"esp"),"xmm1");
&paddq ("xmm2",&QWP(0x60,"esp"));
&paddq ("xmm2","xmm5"); # a[6]*b[7]+lw(a[0]*b[7]), reduction step
&paddq ("xmm3","xmm4"); # a[6]*b[7]+lw(a[0]*b[7])*0xffffffff, reduction step
&movdqa(&QWP(0x50,"esp"),"xmm2");
&movdqa(&QWP(0x60,"esp"),"xmm3");
&movdqa ("xmm1",&QWP(0x10,"esp"));
&movdqa ("xmm2",&QWP(0x20,"esp"));
&movdqa ("xmm3",&QWP(0x30,"esp"));
&movq ("xmm4","xmm0"); # "flatten"
&pand ("xmm0","xmm7");
&xor ("ebp","ebp");
&pslldq ("xmm4",6);
&movq ("xmm5","xmm1");
&paddq ("xmm0","xmm4");
&pand ("xmm1","xmm7");
&psrldq ("xmm0",6);
&movd ("eax","xmm0");
&psrldq ("xmm0",4);
&paddq ("xmm5","xmm0");
&movdqa ("xmm0",&QWP(0x40,"esp"));
&sub ("eax",-1); # start subtracting modulus,
# this is used to determine
# if result is larger/smaller
# than modulus (see below)
&pslldq ("xmm5",6);
&movq ("xmm4","xmm2");
&paddq ("xmm1","xmm5");
&pand ("xmm2","xmm7");
&psrldq ("xmm1",6);
&mov (&DWP(4*0,"edi"),"eax");
&movd ("eax","xmm1");
&psrldq ("xmm1",4);
&paddq ("xmm4","xmm1");
&movdqa ("xmm1",&QWP(0x50,"esp"));
&sbb ("eax",-1);
&pslldq ("xmm4",6);
&movq ("xmm5","xmm3");
&paddq ("xmm2","xmm4");
&pand ("xmm3","xmm7");
&psrldq ("xmm2",6);
&mov (&DWP(4*1,"edi"),"eax");
&movd ("eax","xmm2");
&psrldq ("xmm2",4);
&paddq ("xmm5","xmm2");
&movdqa ("xmm2",&QWP(0x60,"esp"));
&sbb ("eax",-1);
&pslldq ("xmm5",6);
&movq ("xmm4","xmm0");
&paddq ("xmm3","xmm5");
&pand ("xmm0","xmm7");
&psrldq ("xmm3",6);
&mov (&DWP(4*2,"edi"),"eax");
&movd ("eax","xmm3");
&psrldq ("xmm3",4);
&paddq ("xmm4","xmm3");
&sbb ("eax",0);
&pslldq ("xmm4",6);
&movq ("xmm5","xmm1");
&paddq ("xmm0","xmm4");
&pand ("xmm1","xmm7");
&psrldq ("xmm0",6);
&mov (&DWP(4*3,"edi"),"eax");
&movd ("eax","xmm0");
&psrldq ("xmm0",4);
&paddq ("xmm5","xmm0");
&sbb ("eax",0);
&pslldq ("xmm5",6);
&movq ("xmm4","xmm2");
&paddq ("xmm1","xmm5");
&pand ("xmm2","xmm7");
&psrldq ("xmm1",6);
&movd ("ebx","xmm1");
&psrldq ("xmm1",4);
&mov ("esp","edx");
&paddq ("xmm4","xmm1");
&pslldq ("xmm4",6);
&paddq ("xmm2","xmm4");
&psrldq ("xmm2",6);
&movd ("ecx","xmm2");
&psrldq ("xmm2",4);
&sbb ("ebx",0);
&movd ("edx","xmm2");
&pextrw ("esi","xmm2",2); # top-most overflow bit
&sbb ("ecx",1);
&sbb ("edx",-1);
&sbb ("esi",0); # borrow from subtraction
# Final step is "if result > mod, subtract mod", and at this point
# we have result - mod written to output buffer, as well as borrow
# bit from this subtraction, and if borrow bit is set, we add
# modulus back.
#
# Note that because mod has special form, i.e. consists of
# 0xffffffff, 1 and 0s, we can conditionally synthesize it by
# assigning borrow bit to one register, %ebp, and its negative
# to another, %esi. But we started by calculating %esi...
&sub ("ebp","esi");
&add (&DWP(4*0,"edi"),"esi"); # add modulus or zero
&adc (&DWP(4*1,"edi"),"esi");
&adc (&DWP(4*2,"edi"),"esi");
&adc (&DWP(4*3,"edi"),0);
&adc ("eax",0);
&adc ("ebx",0);
&mov (&DWP(4*4,"edi"),"eax");
&adc ("ecx","ebp");
&mov (&DWP(4*5,"edi"),"ebx");
&adc ("edx","esi");
&mov (&DWP(4*6,"edi"),"ecx");
&mov (&DWP(4*7,"edi"),"edx");
&ret ();
&set_label("mul_mont_ialu",16); }
########################################
# IALU code path suitable for all CPUs.
########################################
# stack layout:
# +------------------------------------+< %esp
# | 8 32-bit temporary words, accessed |
# | as circular buffer |
# . .
# . .
# +------------------------------------+< +32
# | offloaded destination pointer |
# +------------------------------------+
# | unused |
# +------------------------------------+< +40
&sub ("esp",10*4);
&mov ("eax",&DWP(0*4,"esi")); # a[0]
&mov ("ebx",&DWP(0*4,"ebp")); # b[0]
&mov (&DWP(8*4,"esp"),"edi"); # off-load dst ptr
&mul ("ebx"); # a[0]*b[0]
&mov (&DWP(0*4,"esp"),"eax"); # t[0]
&mov ("eax",&DWP(1*4,"esi"));
&mov ("ecx","edx")
&mul ("ebx"); # a[1]*b[0]
&add ("ecx","eax");
&mov ("eax",&DWP(2*4,"esi"));
&adc ("edx",0);
&mov (&DWP(1*4,"esp"),"ecx"); # t[1]
&mov ("ecx","edx");
&mul ("ebx"); # a[2]*b[0]
&add ("ecx","eax");
&mov ("eax",&DWP(3*4,"esi"));
&adc ("edx",0);
&mov (&DWP(2*4,"esp"),"ecx"); # t[2]
&mov ("ecx","edx");
&mul ("ebx"); # a[3]*b[0]
&add ("ecx","eax");
&mov ("eax",&DWP(4*4,"esi"));
&adc ("edx",0);
&mov (&DWP(3*4,"esp"),"ecx"); # t[3]
&mov ("ecx","edx");
&mul ("ebx"); # a[4]*b[0]
&add ("ecx","eax");
&mov ("eax",&DWP(5*4,"esi"));
&adc ("edx",0);
&mov (&DWP(4*4,"esp"),"ecx"); # t[4]
&mov ("ecx","edx");
&mul ("ebx"); # a[5]*b[0]
&add ("ecx","eax");
&mov ("eax",&DWP(6*4,"esi"));
&adc ("edx",0);
&mov (&DWP(5*4,"esp"),"ecx"); # t[5]
&mov ("ecx","edx");
&mul ("ebx"); # a[6]*b[0]
&add ("ecx","eax");
&mov ("eax",&DWP(7*4,"esi"));
&adc ("edx",0);
&mov (&DWP(6*4,"esp"),"ecx"); # t[6]
&mov ("ecx","edx");
&xor ("edi","edi"); # initial top-most carry
&mul ("ebx"); # a[7]*b[0]
&add ("ecx","eax"); # t[7]
&mov ("eax",&DWP(0*4,"esp")); # t[0]
&adc ("edx",0); # t[8]
for ($i=0;$i<7;$i++) {
my $j=$i+1;
# Reduction iteration is normally performed by accumulating
# result of multiplication of modulus by "magic" digit [and
# omitting least significant word, which is guaranteed to
# be 0], but thanks to special form of modulus and "magic"
# digit being equal to least significant word, it can be
# performed with additions and subtractions alone. Indeed:
#
# ffff.0001.0000.0000.0000.ffff.ffff.ffff
# * abcd
# + xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
#
# Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
# rewrite above as:
#
# xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
# + abcd.0000.abcd.0000.0000.abcd.0000.0000.0000
# - abcd.0000.0000.0000.0000.0000.0000.abcd
#
# or marking redundant operations:
#
# xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.----
# + abcd.0000.abcd.0000.0000.abcd.----.----.----
# - abcd.----.----.----.----.----.----.----
&add (&DWP((($i+3)%8)*4,"esp"),"eax"); # t[3]+=t[0]
&adc (&DWP((($i+4)%8)*4,"esp"),0); # t[4]+=0
&adc (&DWP((($i+5)%8)*4,"esp"),0); # t[5]+=0
&adc (&DWP((($i+6)%8)*4,"esp"),"eax"); # t[6]+=t[0]
&adc ("ecx",0); # t[7]+=0
&adc ("edx","eax"); # t[8]+=t[0]
&adc ("edi",0); # top-most carry
&mov ("ebx",&DWP($j*4,"ebp")); # b[i]
&sub ("ecx","eax"); # t[7]-=t[0]
&mov ("eax",&DWP(0*4,"esi")); # a[0]
&sbb ("edx",0); # t[8]-=0
&mov (&DWP((($i+7)%8)*4,"esp"),"ecx");
&sbb ("edi",0); # top-most carry,
# keep in mind that
# netto result is
# *addition* of value
# with (abcd<<32)-abcd
# on top, so that
# underflow is
# impossible, because
# (abcd<<32)-abcd
# doesn't underflow
&mov (&DWP((($i+8)%8)*4,"esp"),"edx");
&mul ("ebx"); # a[0]*b[i]
&add ("eax",&DWP((($j+0)%8)*4,"esp"));
&adc ("edx",0);
&mov (&DWP((($j+0)%8)*4,"esp"),"eax");
&mov ("eax",&DWP(1*4,"esi"));
&mov ("ecx","edx")
&mul ("ebx"); # a[1]*b[i]
&add ("ecx",&DWP((($j+1)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax");
&adc ("edx",0);
&mov ("eax",&DWP(2*4,"esi"));
&mov (&DWP((($j+1)%8)*4,"esp"),"ecx");
&mov ("ecx","edx");
&mul ("ebx"); # a[2]*b[i]
&add ("ecx",&DWP((($j+2)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax");
&adc ("edx",0);
&mov ("eax",&DWP(3*4,"esi"));
&mov (&DWP((($j+2)%8)*4,"esp"),"ecx");
&mov ("ecx","edx");
&mul ("ebx"); # a[3]*b[i]
&add ("ecx",&DWP((($j+3)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax");
&adc ("edx",0);
&mov ("eax",&DWP(4*4,"esi"));
&mov (&DWP((($j+3)%8)*4,"esp"),"ecx");
&mov ("ecx","edx");
&mul ("ebx"); # a[4]*b[i]
&add ("ecx",&DWP((($j+4)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax");
&adc ("edx",0);
&mov ("eax",&DWP(5*4,"esi"));
&mov (&DWP((($j+4)%8)*4,"esp"),"ecx");
&mov ("ecx","edx");
&mul ("ebx"); # a[5]*b[i]
&add ("ecx",&DWP((($j+5)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax");
&adc ("edx",0);
&mov ("eax",&DWP(6*4,"esi"));
&mov (&DWP((($j+5)%8)*4,"esp"),"ecx");
&mov ("ecx","edx");
&mul ("ebx"); # a[6]*b[i]
&add ("ecx",&DWP((($j+6)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax");
&adc ("edx",0);
&mov ("eax",&DWP(7*4,"esi"));
&mov (&DWP((($j+6)%8)*4,"esp"),"ecx");
&mov ("ecx","edx");
&mul ("ebx"); # a[7]*b[i]
&add ("ecx",&DWP((($j+7)%8)*4,"esp"));
&adc ("edx",0);
&add ("ecx","eax"); # t[7]
&mov ("eax",&DWP((($j+0)%8)*4,"esp")); # t[0]
&adc ("edx","edi"); # t[8]
&mov ("edi",0);
&adc ("edi",0); # top-most carry
}
&mov ("ebp",&DWP(8*4,"esp")); # restore dst ptr
&xor ("esi","esi");
my $j=$i+1;
# last multiplication-less reduction
&add (&DWP((($i+3)%8)*4,"esp"),"eax"); # t[3]+=t[0]
&adc (&DWP((($i+4)%8)*4,"esp"),0); # t[4]+=0
&adc (&DWP((($i+5)%8)*4,"esp"),0); # t[5]+=0
&adc (&DWP((($i+6)%8)*4,"esp"),"eax"); # t[6]+=t[0]
&adc ("ecx",0); # t[7]+=0
&adc ("edx","eax"); # t[8]+=t[0]
&adc ("edi",0); # top-most carry
&mov ("ebx",&DWP((($j+1)%8)*4,"esp"));
&sub ("ecx","eax"); # t[7]-=t[0]
&mov ("eax",&DWP((($j+0)%8)*4,"esp"));
&sbb ("edx",0); # t[8]-=0
&mov (&DWP((($i+7)%8)*4,"esp"),"ecx");
&sbb ("edi",0); # top-most carry
&mov (&DWP((($i+8)%8)*4,"esp"),"edx");
# Final step is "if result > mod, subtract mod", but we do it
# "other way around", namely write result - mod to output buffer
# and if subtraction borrowed, add modulus back.
&mov ("ecx",&DWP((($j+2)%8)*4,"esp"));
&sub ("eax",-1);
&mov ("edx",&DWP((($j+3)%8)*4,"esp"));
&sbb ("ebx",-1);
&mov (&DWP(0*4,"ebp"),"eax");
&sbb ("ecx",-1);
&mov (&DWP(1*4,"ebp"),"ebx");
&sbb ("edx",0);
&mov (&DWP(2*4,"ebp"),"ecx");
&mov (&DWP(3*4,"ebp"),"edx");
&mov ("eax",&DWP((($j+4)%8)*4,"esp"));
&mov ("ebx",&DWP((($j+5)%8)*4,"esp"));
&mov ("ecx",&DWP((($j+6)%8)*4,"esp"));
&sbb ("eax",0);
&mov ("edx",&DWP((($j+7)%8)*4,"esp"));
&sbb ("ebx",0);
&sbb ("ecx",1);
&sbb ("edx",-1);
&sbb ("edi",0);
# Note that because mod has special form, i.e. consists of
# 0xffffffff, 1 and 0s, we can conditionally synthesize it by
# assigning borrow bit to one register, %ebp, and its negative
# to another, %esi. But we started by calculating %esi...
&sub ("esi","edi");
&add (&DWP(0*4,"ebp"),"edi"); # add modulus or zero
&adc (&DWP(1*4,"ebp"),"edi");
&adc (&DWP(2*4,"ebp"),"edi");
&adc (&DWP(3*4,"ebp"),0);
&adc ("eax",0);
&adc ("ebx",0);
&mov (&DWP(4*4,"ebp"),"eax");
&adc ("ecx","esi");
&mov (&DWP(5*4,"ebp"),"ebx");
&adc ("edx","edi");
&mov (&DWP(6*4,"ebp"),"ecx");
&mov ("edi","ebp"); # fulfill contract
&mov (&DWP(7*4,"ebp"),"edx");
&add ("esp",10*4);
&ret ();
&function_end_B("_ecp_nistz256_mul_mont");
########################################################################
# void ecp_nistz256_scatter_w5(void *edi,const P256_POINT *esi,
# int ebp);
&function_begin("ecp_nistz256_scatter_w5");
&mov ("edi",&wparam(0));
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
&lea ("edi",&DWP(128-4,"edi","ebp",4));
&mov ("ebp",96/16);
&set_label("scatter_w5_loop");
&mov ("eax",&DWP(0,"esi"));
&mov ("ebx",&DWP(4,"esi"));
&mov ("ecx",&DWP(8,"esi"));
&mov ("edx",&DWP(12,"esi"));
&lea ("esi",&DWP(16,"esi"));
&mov (&DWP(64*0-128,"edi"),"eax");
&mov (&DWP(64*1-128,"edi"),"ebx");
&mov (&DWP(64*2-128,"edi"),"ecx");
&mov (&DWP(64*3-128,"edi"),"edx");
&lea ("edi",&DWP(64*4,"edi"));
&dec ("ebp");
&jnz (&label("scatter_w5_loop"));
&function_end("ecp_nistz256_scatter_w5");
########################################################################
# void ecp_nistz256_gather_w5(P256_POINT *edi,const void *esi,
# int ebp);
&function_begin("ecp_nistz256_gather_w5");
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
&lea ("esi",&DWP(0,"esi","ebp",4));
&neg ("ebp");
&sar ("ebp",31);
&mov ("edi",&wparam(0));
&lea ("esi",&DWP(0,"esi","ebp",4));
for($i=0;$i<24;$i+=4) {
&mov ("eax",&DWP(64*($i+0),"esi"));
&mov ("ebx",&DWP(64*($i+1),"esi"));
&mov ("ecx",&DWP(64*($i+2),"esi"));
&mov ("edx",&DWP(64*($i+3),"esi"));
&and ("eax","ebp");
&and ("ebx","ebp");
&and ("ecx","ebp");
&and ("edx","ebp");
&mov (&DWP(4*($i+0),"edi"),"eax");
&mov (&DWP(4*($i+1),"edi"),"ebx");
&mov (&DWP(4*($i+2),"edi"),"ecx");
&mov (&DWP(4*($i+3),"edi"),"edx");
}
&function_end("ecp_nistz256_gather_w5");
########################################################################
# void ecp_nistz256_scatter_w7(void *edi,const P256_POINT_AFFINE *esi,
# int ebp);
&function_begin("ecp_nistz256_scatter_w7");
&mov ("edi",&wparam(0));
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
ec/asm/ecp_nistz256-{!x86_64}.pl: fix scatter_w7 function. The ecp_nistz256_scatter_w7 function is called when application attempts to use custom generator, i.e. rarely. Even though non-x86_64 versions were wrong, it didn't affect point operations, they were just not as fast as expected. Reviewed-by: Rich Salz <rsalz@openssl.org> (Merged from https://github.com/openssl/openssl/pull/6738)
2018-07-18 13:14:44 +00:00
&lea ("edi",&DWP(0,"edi","ebp"));
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov ("ebp",64/4);
&set_label("scatter_w7_loop");
&mov ("eax",&DWP(0,"esi"));
&lea ("esi",&DWP(4,"esi"));
&mov (&BP(64*0,"edi"),"al");
&mov (&BP(64*1,"edi"),"ah");
&shr ("eax",16);
&mov (&BP(64*2,"edi"),"al");
&mov (&BP(64*3,"edi"),"ah");
&lea ("edi",&DWP(64*4,"edi"));
&dec ("ebp");
&jnz (&label("scatter_w7_loop"));
&function_end("ecp_nistz256_scatter_w7");
########################################################################
# void ecp_nistz256_gather_w7(P256_POINT_AFFINE *edi,const void *esi,
# int ebp);
&function_begin("ecp_nistz256_gather_w7");
&mov ("esi",&wparam(1));
&mov ("ebp",&wparam(2));
&add ("esi","ebp");
&neg ("ebp"),
&sar ("ebp",31);
&mov ("edi",&wparam(0));
&lea ("esi",&DWP(0,"esi","ebp"));
for($i=0;$i<64;$i+=4) {
&movz ("eax",&BP(64*($i+0),"esi"));
&movz ("ebx",&BP(64*($i+1),"esi"));
&movz ("ecx",&BP(64*($i+2),"esi"));
&and ("eax","ebp");
&movz ("edx",&BP(64*($i+3),"esi"));
&and ("ebx","ebp");
&mov (&BP($i+0,"edi"),"al");
&and ("ecx","ebp");
&mov (&BP($i+1,"edi"),"bl");
&and ("edx","ebp");
&mov (&BP($i+2,"edi"),"cl");
&mov (&BP($i+3,"edi"),"dl");
}
&function_end("ecp_nistz256_gather_w7");
########################################################################
# following subroutines are "literal" implementation of those found in
# ecp_nistz256.c
#
########################################################################
# void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
#
ec/asm/ecp_nistz256-*.pl: get corner case logic right. RT#4284 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-02-21 20:04:26 +00:00
&static_label("point_double_shortcut");
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&function_begin("ecp_nistz256_point_double");
{ my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
&mov ("esi",&wparam(1));
# above map() describes stack layout with 5 temporary
# 256-bit vectors on top, then we take extra word for
Fix some typos in comments Reviewed-by: Richard Levitte <levitte@openssl.org>
2016-01-06 20:28:39 +00:00
# OPENSSL_ia32cap_P copy.
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&stack_push(8*5+1);
if ($sse2) {
&call ("_picup_eax");
&set_label("pic");
&picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("ebp",&DWP(0,"edx")); }
ec/asm/ecp_nistz256-*.pl: get corner case logic right. RT#4284 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-02-21 20:04:26 +00:00
&set_label("point_double_shortcut");
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov ("eax",&DWP(0,"esi")); # copy in_x
&mov ("ebx",&DWP(4,"esi"));
&mov ("ecx",&DWP(8,"esi"));
&mov ("edx",&DWP(12,"esi"));
&mov (&DWP($in_x+0,"esp"),"eax");
&mov (&DWP($in_x+4,"esp"),"ebx");
&mov (&DWP($in_x+8,"esp"),"ecx");
&mov (&DWP($in_x+12,"esp"),"edx");
&mov ("eax",&DWP(16,"esi"));
&mov ("ebx",&DWP(20,"esi"));
&mov ("ecx",&DWP(24,"esi"));
&mov ("edx",&DWP(28,"esi"));
&mov (&DWP($in_x+16,"esp"),"eax");
&mov (&DWP($in_x+20,"esp"),"ebx");
&mov (&DWP($in_x+24,"esp"),"ecx");
&mov (&DWP($in_x+28,"esp"),"edx");
&mov (&DWP(32*5,"esp"),"ebp"); # OPENSSL_ia32cap_P copy
&lea ("ebp",&DWP(32,"esi"));
&lea ("esi",&DWP(32,"esi"));
&lea ("edi",&DWP($S,"esp"));
&call ("_ecp_nistz256_add"); # p256_mul_by_2(S, in_y);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&mov ("esi",64);
&add ("esi",&wparam(1));
&lea ("edi",&DWP($Zsqr,"esp"));
&mov ("ebp","esi");
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Zsqr, in_z);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($S,"esp"));
&lea ("ebp",&DWP($S,"esp"));
&lea ("edi",&DWP($S,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(S, S);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&mov ("ebp",&wparam(1));
&lea ("esi",&DWP(32,"ebp"));
&lea ("ebp",&DWP(64,"ebp"));
&lea ("edi",&DWP($tmp0,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(tmp0, in_z, in_y);
&lea ("esi",&DWP($in_x,"esp"));
&lea ("ebp",&DWP($Zsqr,"esp"));
&lea ("edi",&DWP($M,"esp"));
&call ("_ecp_nistz256_add"); # p256_add(M, in_x, Zsqr);
&mov ("edi",64);
&lea ("esi",&DWP($tmp0,"esp"));
&lea ("ebp",&DWP($tmp0,"esp"));
&add ("edi",&wparam(0));
&call ("_ecp_nistz256_add"); # p256_mul_by_2(res_z, tmp0);
&lea ("esi",&DWP($in_x,"esp"));
&lea ("ebp",&DWP($Zsqr,"esp"));
&lea ("edi",&DWP($Zsqr,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(Zsqr, in_x, Zsqr);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($S,"esp"));
&lea ("ebp",&DWP($S,"esp"));
&lea ("edi",&DWP($tmp0,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(tmp0, S);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($M,"esp"));
&lea ("ebp",&DWP($Zsqr,"esp"));
&lea ("edi",&DWP($M,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(M, M, Zsqr);
&mov ("edi",32);
&lea ("esi",&DWP($tmp0,"esp"));
&add ("edi",&wparam(0));
&call ("_ecp_nistz256_div_by_2"); # p256_div_by_2(res_y, tmp0);
&lea ("esi",&DWP($M,"esp"));
&lea ("ebp",&DWP($M,"esp"));
&lea ("edi",&DWP($tmp0,"esp"));
&call ("_ecp_nistz256_add"); # 1/2 p256_mul_by_3(M, M);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in_x,"esp"));
&lea ("ebp",&DWP($S,"esp"));
&lea ("edi",&DWP($S,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S, S, in_x);
&lea ("esi",&DWP($tmp0,"esp"));
&lea ("ebp",&DWP($M,"esp"));
&lea ("edi",&DWP($M,"esp"));
&call ("_ecp_nistz256_add"); # 2/2 p256_mul_by_3(M, M);
&lea ("esi",&DWP($S,"esp"));
&lea ("ebp",&DWP($S,"esp"));
&lea ("edi",&DWP($tmp0,"esp"));
&call ("_ecp_nistz256_add"); # p256_mul_by_2(tmp0, S);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($M,"esp"));
&lea ("ebp",&DWP($M,"esp"));
&mov ("edi",&wparam(0));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(res_x, M);
&mov ("esi","edi"); # %edi is still res_x here
&lea ("ebp",&DWP($tmp0,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_x, res_x, tmp0);
&lea ("esi",&DWP($S,"esp"));
&mov ("ebp","edi"); # %edi is still res_x
&lea ("edi",&DWP($S,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(S, S, res_x);
&mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
&mov ("esi","edi"); # %edi is still &S
&lea ("ebp",&DWP($M,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S, S, M);
&mov ("ebp",32);
&lea ("esi",&DWP($S,"esp"));
&add ("ebp",&wparam(0));
&mov ("edi","ebp");
&call ("_ecp_nistz256_sub"); # p256_sub(res_y, S, res_y);
&stack_pop(8*5+1);
} &function_end("ecp_nistz256_point_double");
########################################################################
# void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
# const P256_POINT *in2);
&function_begin("ecp_nistz256_point_add");
{ my ($res_x,$res_y,$res_z,
$in1_x,$in1_y,$in1_z,
$in2_x,$in2_y,$in2_z,
$H,$Hsqr,$R,$Rsqr,$Hcub,
$U1,$U2,$S1,$S2)=map(32*$_,(0..17));
my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
&mov ("esi",&wparam(2));
# above map() describes stack layout with 18 temporary
# 256-bit vectors on top, then we take extra words for
# !in1infty, !in2infty, result of check for zero and
Fix some typos in comments Reviewed-by: Richard Levitte <levitte@openssl.org>
2016-01-06 20:28:39 +00:00
# OPENSSL_ia32cap_P copy. [one unused word for padding]
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&stack_push(8*18+5);
if ($sse2) {
&call ("_picup_eax");
&set_label("pic");
&picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("ebp",&DWP(0,"edx")); }
&lea ("edi",&DWP($in2_x,"esp"));
for($i=0;$i<96;$i+=16) {
&mov ("eax",&DWP($i+0,"esi")); # copy in2
&mov ("ebx",&DWP($i+4,"esi"));
&mov ("ecx",&DWP($i+8,"esi"));
&mov ("edx",&DWP($i+12,"esi"));
&mov (&DWP($i+0,"edi"),"eax");
&mov (&DWP(32*18+12,"esp"),"ebp") if ($i==0);
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&mov ("ebp","eax") if ($i==64);
&or ("ebp","eax") if ($i>64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+4,"edi"),"ebx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","ebx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+8,"edi"),"ecx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","ecx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+12,"edi"),"edx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","edx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
}
&xor ("eax","eax");
&mov ("esi",&wparam(1));
&sub ("eax","ebp");
&or ("ebp","eax");
&sar ("ebp",31);
&mov (&DWP(32*18+4,"esp"),"ebp"); # !in2infty
&lea ("edi",&DWP($in1_x,"esp"));
for($i=0;$i<96;$i+=16) {
&mov ("eax",&DWP($i+0,"esi")); # copy in1
&mov ("ebx",&DWP($i+4,"esi"));
&mov ("ecx",&DWP($i+8,"esi"));
&mov ("edx",&DWP($i+12,"esi"));
&mov (&DWP($i+0,"edi"),"eax");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&mov ("ebp","eax") if ($i==64);
&or ("ebp","eax") if ($i>64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+4,"edi"),"ebx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","ebx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+8,"edi"),"ecx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","ecx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+12,"edi"),"edx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","edx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
}
&xor ("eax","eax");
&sub ("eax","ebp");
&or ("ebp","eax");
&sar ("ebp",31);
&mov (&DWP(32*18+0,"esp"),"ebp"); # !in1infty
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in2_z,"esp"));
&lea ("ebp",&DWP($in2_z,"esp"));
&lea ("edi",&DWP($Z2sqr,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Z2sqr, in2_z);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in1_z,"esp"));
&lea ("ebp",&DWP($in1_z,"esp"));
&lea ("edi",&DWP($Z1sqr,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Z1sqr, in1_z);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($Z2sqr,"esp"));
&lea ("ebp",&DWP($in2_z,"esp"));
&lea ("edi",&DWP($S1,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S1, Z2sqr, in2_z);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($Z1sqr,"esp"));
&lea ("ebp",&DWP($in1_z,"esp"));
&lea ("edi",&DWP($S2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, Z1sqr, in1_z);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in1_y,"esp"));
&lea ("ebp",&DWP($S1,"esp"));
&lea ("edi",&DWP($S1,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S1, S1, in1_y);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in2_y,"esp"));
&lea ("ebp",&DWP($S2,"esp"));
&lea ("edi",&DWP($S2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, S2, in2_y);
&lea ("esi",&DWP($S2,"esp"));
&lea ("ebp",&DWP($S1,"esp"));
&lea ("edi",&DWP($R,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(R, S2, S1);
&or ("ebx","eax"); # see if result is zero
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&or ("ebx","ecx");
&or ("ebx","edx");
&or ("ebx",&DWP(0,"edi"));
&or ("ebx",&DWP(4,"edi"));
&lea ("esi",&DWP($in1_x,"esp"));
&or ("ebx",&DWP(8,"edi"));
&lea ("ebp",&DWP($Z2sqr,"esp"));
&or ("ebx",&DWP(12,"edi"));
&lea ("edi",&DWP($U1,"esp"));
&mov (&DWP(32*18+8,"esp"),"ebx");
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U1, in1_x, Z2sqr);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in2_x,"esp"));
&lea ("ebp",&DWP($Z1sqr,"esp"));
&lea ("edi",&DWP($U2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, in2_x, Z1sqr);
&lea ("esi",&DWP($U2,"esp"));
&lea ("ebp",&DWP($U1,"esp"));
&lea ("edi",&DWP($H,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(H, U2, U1);
&or ("eax","ebx"); # see if result is zero
&or ("eax","ecx");
&or ("eax","edx");
&or ("eax",&DWP(0,"edi"));
&or ("eax",&DWP(4,"edi"));
&or ("eax",&DWP(8,"edi"));
&or ("eax",&DWP(12,"edi"));
&data_byte(0x3e); # predict taken
&jnz (&label("add_proceed")); # is_equal(U1,U2)?
&mov ("eax",&DWP(32*18+0,"esp"));
&and ("eax",&DWP(32*18+4,"esp"));
&mov ("ebx",&DWP(32*18+8,"esp"));
&jz (&label("add_proceed")); # (in1infty || in2infty)?
&test ("ebx","ebx");
ec/asm/ecp_nistz256-*.pl: get corner case logic right. RT#4284 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-02-21 20:04:26 +00:00
&jz (&label("add_double")); # is_equal(S1,S2)?
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov ("edi",&wparam(0));
&xor ("eax","eax");
&mov ("ecx",96/4);
&data_byte(0xfc,0xf3,0xab); # cld; stosd
&jmp (&label("add_done"));
ec/asm/ecp_nistz256-*.pl: get corner case logic right. RT#4284 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-02-21 20:04:26 +00:00
&set_label("add_double",16);
&mov ("esi",&wparam(1));
&mov ("ebp",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&add ("esp",4*((8*18+5)-(8*5+1))); # difference in frame sizes
&jmp (&label("point_double_shortcut"));
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&set_label("add_proceed",16);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($R,"esp"));
&lea ("ebp",&DWP($R,"esp"));
&lea ("edi",&DWP($Rsqr,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Rsqr, R);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($H,"esp"));
&lea ("ebp",&DWP($in1_z,"esp"));
&lea ("edi",&DWP($res_z,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_z, H, in1_z);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($H,"esp"));
&lea ("ebp",&DWP($H,"esp"));
&lea ("edi",&DWP($Hsqr,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Hsqr, H);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in2_z,"esp"));
&lea ("ebp",&DWP($res_z,"esp"));
&lea ("edi",&DWP($res_z,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_z, res_z, in2_z);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($Hsqr,"esp"));
&lea ("ebp",&DWP($U1,"esp"));
&lea ("edi",&DWP($U2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, U1, Hsqr);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($H,"esp"));
&lea ("ebp",&DWP($Hsqr,"esp"));
&lea ("edi",&DWP($Hcub,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(Hcub, Hsqr, H);
&lea ("esi",&DWP($U2,"esp"));
&lea ("ebp",&DWP($U2,"esp"));
&lea ("edi",&DWP($Hsqr,"esp"));
&call ("_ecp_nistz256_add"); # p256_mul_by_2(Hsqr, U2);
&lea ("esi",&DWP($Rsqr,"esp"));
&lea ("ebp",&DWP($Hsqr,"esp"));
&lea ("edi",&DWP($res_x,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_x, Rsqr, Hsqr);
&lea ("esi",&DWP($res_x,"esp"));
&lea ("ebp",&DWP($Hcub,"esp"));
&lea ("edi",&DWP($res_x,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_x, res_x, Hcub);
&lea ("esi",&DWP($U2,"esp"));
&lea ("ebp",&DWP($res_x,"esp"));
&lea ("edi",&DWP($res_y,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_y, U2, res_x);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($Hcub,"esp"));
&lea ("ebp",&DWP($S1,"esp"));
&lea ("edi",&DWP($S2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, S1, Hcub);
&mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($R,"esp"));
&lea ("ebp",&DWP($res_y,"esp"));
&lea ("edi",&DWP($res_y,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_y, R, res_y);
&lea ("esi",&DWP($res_y,"esp"));
&lea ("ebp",&DWP($S2,"esp"));
&lea ("edi",&DWP($res_y,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_y, res_y, S2);
&mov ("ebp",&DWP(32*18+0,"esp")); # !in1infty
&mov ("esi",&DWP(32*18+4,"esp")); # !in2infty
&mov ("edi",&wparam(0));
&mov ("edx","ebp");
&not ("ebp");
&and ("edx","esi");
&and ("ebp","esi");
&not ("esi");
########################################
# conditional moves
for($i=64;$i<96;$i+=4) {
&mov ("eax","edx");
&and ("eax",&DWP($res_x+$i,"esp"));
&mov ("ebx","ebp");
&and ("ebx",&DWP($in2_x+$i,"esp"));
&mov ("ecx","esi");
&and ("ecx",&DWP($in1_x+$i,"esp"));
&or ("eax","ebx");
&or ("eax","ecx");
&mov (&DWP($i,"edi"),"eax");
}
for($i=0;$i<64;$i+=4) {
&mov ("eax","edx");
&and ("eax",&DWP($res_x+$i,"esp"));
&mov ("ebx","ebp");
&and ("ebx",&DWP($in2_x+$i,"esp"));
&mov ("ecx","esi");
&and ("ecx",&DWP($in1_x+$i,"esp"));
&or ("eax","ebx");
&or ("eax","ecx");
&mov (&DWP($i,"edi"),"eax");
}
&set_label("add_done");
&stack_pop(8*18+5);
} &function_end("ecp_nistz256_point_add");
########################################################################
# void ecp_nistz256_point_add_affine(P256_POINT *out,
# const P256_POINT *in1,
# const P256_POINT_AFFINE *in2);
&function_begin("ecp_nistz256_point_add_affine");
{
my ($res_x,$res_y,$res_z,
$in1_x,$in1_y,$in1_z,
$in2_x,$in2_y,
$U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..14));
my $Z1sqr = $S2;
my @ONE_mont=(1,0,0,-1,-1,-1,-2,0);
&mov ("esi",&wparam(1));
# above map() describes stack layout with 15 temporary
# 256-bit vectors on top, then we take extra words for
Fix some typos in comments Reviewed-by: Richard Levitte <levitte@openssl.org>
2016-01-06 20:28:39 +00:00
# !in1infty, !in2infty, and OPENSSL_ia32cap_P copy.
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&stack_push(8*15+3);
if ($sse2) {
&call ("_picup_eax");
&set_label("pic");
&picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
&mov ("ebp",&DWP(0,"edx")); }
&lea ("edi",&DWP($in1_x,"esp"));
for($i=0;$i<96;$i+=16) {
&mov ("eax",&DWP($i+0,"esi")); # copy in1
&mov ("ebx",&DWP($i+4,"esi"));
&mov ("ecx",&DWP($i+8,"esi"));
&mov ("edx",&DWP($i+12,"esi"));
&mov (&DWP($i+0,"edi"),"eax");
&mov (&DWP(32*15+8,"esp"),"ebp") if ($i==0);
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&mov ("ebp","eax") if ($i==64);
&or ("ebp","eax") if ($i>64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+4,"edi"),"ebx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","ebx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+8,"edi"),"ecx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","ecx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
&mov (&DWP($i+12,"edi"),"edx");
ec/ecp_nistz256: harmonize is_infinity with ec_GFp_simple_is_at_infinity. RT#4625 Reviewed-by: Rich Salz <rsalz@openssl.org>
2016-08-19 21:18:35 +00:00
&or ("ebp","edx") if ($i>=64);
Add ec/asm/ecp_nistz256-x86.pl module. Reviewed-by: Emilia Käsper <emilia@openssl.org>
2015-02-09 22:19:16 +00:00
}
&xor ("eax","eax");
&mov ("esi",&wparam(2));
&sub ("eax","ebp");
&or ("ebp","eax");
&sar ("ebp",31);
&mov (&DWP(32*15+0,"esp"),"ebp"); # !in1infty
&lea ("edi",&DWP($in2_x,"esp"));
for($i=0;$i<64;$i+=16) {
&mov ("eax",&DWP($i+0,"esi")); # copy in2
&mov ("ebx",&DWP($i+4,"esi"));
&mov ("ecx",&DWP($i+8,"esi"));
&mov ("edx",&DWP($i+12,"esi"));
&mov (&DWP($i+0,"edi"),"eax");
&mov ("ebp","eax") if ($i==0);
&or ("ebp","eax") if ($i!=0);
&mov (&DWP($i+4,"edi"),"ebx");
&or ("ebp","ebx");
&mov (&DWP($i+8,"edi"),"ecx");
&or ("ebp","ecx");
&mov (&DWP($i+12,"edi"),"edx");
&or ("ebp","edx");
}
&xor ("ebx","ebx");
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&sub ("ebx","ebp");
&lea ("esi",&DWP($in1_z,"esp"));
&or ("ebx","ebp");
&lea ("ebp",&DWP($in1_z,"esp"));
&sar ("ebx",31);
&lea ("edi",&DWP($Z1sqr,"esp"));
&mov (&DWP(32*15+4,"esp"),"ebx"); # !in2infty
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Z1sqr, in1_z);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in2_x,"esp"));
&mov ("ebp","edi"); # %esi is stull &Z1sqr
&lea ("edi",&DWP($U2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, Z1sqr, in2_x);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in1_z,"esp"));
&lea ("ebp",&DWP($Z1sqr,"esp"));
&lea ("edi",&DWP($S2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, Z1sqr, in1_z);
&lea ("esi",&DWP($U2,"esp"));
&lea ("ebp",&DWP($in1_x,"esp"));
&lea ("edi",&DWP($H,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(H, U2, in1_x);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in2_y,"esp"));
&lea ("ebp",&DWP($S2,"esp"));
&lea ("edi",&DWP($S2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, S2, in2_y);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in1_z,"esp"));
&lea ("ebp",&DWP($H,"esp"));
&lea ("edi",&DWP($res_z,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_z, H, in1_z);
&lea ("esi",&DWP($S2,"esp"));
&lea ("ebp",&DWP($in1_y,"esp"));
&lea ("edi",&DWP($R,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(R, S2, in1_y);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($H,"esp"));
&lea ("ebp",&DWP($H,"esp"));
&lea ("edi",&DWP($Hsqr,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Hsqr, H);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($R,"esp"));
&lea ("ebp",&DWP($R,"esp"));
&lea ("edi",&DWP($Rsqr,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Rsqr, R);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($in1_x,"esp"));
&lea ("ebp",&DWP($Hsqr,"esp"));
&lea ("edi",&DWP($U2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, in1_x, Hsqr);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($H,"esp"));
&lea ("ebp",&DWP($Hsqr,"esp"));
&lea ("edi",&DWP($Hcub,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(Hcub, Hsqr, H);
&lea ("esi",&DWP($U2,"esp"));
&lea ("ebp",&DWP($U2,"esp"));
&lea ("edi",&DWP($Hsqr,"esp"));
&call ("_ecp_nistz256_add"); # p256_mul_by_2(Hsqr, U2);
&lea ("esi",&DWP($Rsqr,"esp"));
&lea ("ebp",&DWP($Hsqr,"esp"));
&lea ("edi",&DWP($res_x,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_x, Rsqr, Hsqr);
&lea ("esi",&DWP($res_x,"esp"));
&lea ("ebp",&DWP($Hcub,"esp"));
&lea ("edi",&DWP($res_x,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_x, res_x, Hcub);
&lea ("esi",&DWP($U2,"esp"));
&lea ("ebp",&DWP($res_x,"esp"));
&lea ("edi",&DWP($res_y,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_y, U2, res_x);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($Hcub,"esp"));
&lea ("ebp",&DWP($in1_y,"esp"));
&lea ("edi",&DWP($S2,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, Hcub, in1_y);
&mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
&lea ("esi",&DWP($R,"esp"));
&lea ("ebp",&DWP($res_y,"esp"));
&lea ("edi",&DWP($res_y,"esp"));
&call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_y, res_y, R);
&lea ("esi",&DWP($res_y,"esp"));
&lea ("ebp",&DWP($S2,"esp"));
&lea ("edi",&DWP($res_y,"esp"));
&call ("_ecp_nistz256_sub"); # p256_sub(res_y, res_y, S2);
&mov ("ebp",&DWP(32*15+0,"esp")); # !in1infty
&mov ("esi",&DWP(32*15+4,"esp")); # !in2infty
&mov ("edi",&wparam(0));
&mov ("edx","ebp");
&not ("ebp");
&and ("edx","esi");
&and ("ebp","esi");
&not ("esi");
########################################
# conditional moves
for($i=64;$i<96;$i+=4) {
my $one=@ONE_mont[($i-64)/4];
&mov ("eax","edx");
&and ("eax",&DWP($res_x+$i,"esp"));
&mov ("ebx","ebp") if ($one && $one!=-1);
&and ("ebx",$one) if ($one && $one!=-1);
&mov ("ecx","esi");
&and ("ecx",&DWP($in1_x+$i,"esp"));
&or ("eax",$one==-1?"ebp":"ebx") if ($one);
&or ("eax","ecx");
&mov (&DWP($i,"edi"),"eax");
}
for($i=0;$i<64;$i+=4) {
&mov ("eax","edx");
&and ("eax",&DWP($res_x+$i,"esp"));
&mov ("ebx","ebp");
&and ("ebx",&DWP($in2_x+$i,"esp"));
&mov ("ecx","esi");
&and ("ecx",&DWP($in1_x+$i,"esp"));
&or ("eax","ebx");
&or ("eax","ecx");
&mov (&DWP($i,"edi"),"eax");
}
&stack_pop(8*15+3);
} &function_end("ecp_nistz256_point_add_affine");
&asm_finish();
Unified - adapt the generation of ec assembler to use GENERATE This gets rid of the BEGINRAW..ENDRAW sections in crypto/ec/build.info. This also moves the assembler generating perl scripts to take the output file name as last command line argument, where necessary. Reviewed-by: Andy Polyakov <appro@openssl.org>
2016-03-07 14:47:09 +00:00
close STDOUT;
Reference in a new issue Copy permalink