3 ##############################################################################
5 # Copyright 2014 Intel Corporation #
7 # Licensed under the Apache License, Version 2.0 (the "License"); #
8 # you may not use this file except in compliance with the License. #
9 # You may obtain a copy of the License at #
11 # http://www.apache.org/licenses/LICENSE-2.0 #
13 # Unless required by applicable law or agreed to in writing, software #
14 # distributed under the License is distributed on an "AS IS" BASIS, #
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
16 # See the License for the specific language governing permissions and #
17 # limitations under the License. #
19 ##############################################################################
21 # Developers and authors: #
22 # Shay Gueron (1, 2), and Vlad Krasnov (1) #
23 # (1) Intel Corporation, Israel Development Center #
24 # (2) University of Haifa #
26 # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with#
29 ##############################################################################
31 # Further optimization by <appro@openssl.org>:
33 # this/original with/without -DECP_NISTZ256_ASM(*)
34 # Opteron +12-49% +110-150%
35 # Bulldozer +14-45% +175-210%
37 # Westmere +12-34% +80-87%
38 # Sandy Bridge +9-35% +110-120%
39 # Ivy Bridge +9-35% +110-125%
40 # Haswell +8-37% +140-160%
41 # Broadwell +18-58% +145-210%
42 # Atom +15-50% +130-180%
43 # VIA Nano +43-160% +300-480%
45 # (*) "without -DECP_NISTZ256_ASM" refers to build with
46 # "enable-ec_nistp_64_gcc_128";
48 # Ranges denote minimum and maximum improvement coefficients depending
49 # on benchmark. Lower coefficients are for ECDSA sign, relatively fastest
50 # server-side operation. Keep in mind that +100% means 2x improvement.
54 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
56 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
58 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
59 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
60 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
61 die "can't locate x86_64-xlate.pl";
63 open OUT,"| \"$^X\" $xlate $flavour $output";
66 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
67 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
68 $avx = ($1>=2.19) + ($1>=2.22);
72 if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
73 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
74 $avx = ($1>=2.09) + ($1>=2.10);
78 if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
79 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
80 $avx = ($1>=10) + ($1>=11);
84 if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9])\.([0-9]+)/) {
85 my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
86 $avx = ($ver>=3.0) + ($ver>=3.01);
92 .extern OPENSSL_ia32cap_P
97 .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
99 # 2^512 mod P precomputed for NIST P256 polynomial
101 .quad 0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe, 0x00000004fffffffd
104 .long 1,1,1,1,1,1,1,1
106 .long 2,2,2,2,2,2,2,2
108 .long 3,3,3,3,3,3,3,3
110 .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
114 ################################################################################
115 # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
117 my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
118 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
119 my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
123 .globl ecp_nistz256_mul_by_2
124 .type ecp_nistz256_mul_by_2,\@function,2
126 ecp_nistz256_mul_by_2:
133 add $a0, $a0 # a0:a3+a0:a3
137 lea .Lpoly(%rip), $a_ptr
164 .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
166 ################################################################################
167 # void ecp_nistz256_div_by_2(uint64_t res[4], uint64_t a[4]);
168 .globl ecp_nistz256_div_by_2
169 .type ecp_nistz256_div_by_2,\@function,2
171 ecp_nistz256_div_by_2:
180 lea .Lpoly(%rip), $a_ptr
191 xor $a_ptr, $a_ptr # borrow $a_ptr
200 mov $a1, $t0 # a0:a3>>1
224 .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
226 ################################################################################
227 # void ecp_nistz256_mul_by_3(uint64_t res[4], uint64_t a[4]);
228 .globl ecp_nistz256_mul_by_3
229 .type ecp_nistz256_mul_by_3,\@function,2
231 ecp_nistz256_mul_by_3:
238 add $a0, $a0 # a0:a3+a0:a3
250 sbb .Lpoly+8*1(%rip), $a1
253 sbb .Lpoly+8*3(%rip), $a3
262 add 8*0($a_ptr), $a0 # a0:a3+=a_ptr[0:3]
272 sbb .Lpoly+8*1(%rip), $a1
275 sbb .Lpoly+8*3(%rip), $a3
290 .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
292 ################################################################################
293 # void ecp_nistz256_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
294 .globl ecp_nistz256_add
295 .type ecp_nistz256_add,\@function,3
306 lea .Lpoly(%rip), $a_ptr
336 .size ecp_nistz256_add,.-ecp_nistz256_add
338 ################################################################################
339 # void ecp_nistz256_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
340 .globl ecp_nistz256_sub
341 .type ecp_nistz256_sub,\@function,3
352 lea .Lpoly(%rip), $a_ptr
382 .size ecp_nistz256_sub,.-ecp_nistz256_sub
384 ################################################################################
385 # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
386 .globl ecp_nistz256_neg
387 .type ecp_nistz256_neg,\@function,2
404 lea .Lpoly(%rip), $a_ptr
428 .size ecp_nistz256_neg,.-ecp_nistz256_neg
432 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
433 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
434 my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
435 my ($poly1,$poly3)=($acc6,$acc7);
438 ################################################################################
439 # void ecp_nistz256_to_mont(
442 .globl ecp_nistz256_to_mont
443 .type ecp_nistz256_to_mont,\@function,2
445 ecp_nistz256_to_mont:
447 $code.=<<___ if ($addx);
449 and OPENSSL_ia32cap_P+8(%rip), %ecx
452 lea .LRR(%rip), $b_org
454 .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
456 ################################################################################
457 # void ecp_nistz256_mul_mont(
462 .globl ecp_nistz256_mul_mont
463 .type ecp_nistz256_mul_mont,\@function,3
465 ecp_nistz256_mul_mont:
467 $code.=<<___ if ($addx);
469 and OPENSSL_ia32cap_P+8(%rip), %ecx
480 $code.=<<___ if ($addx);
486 mov 8*0($b_org), %rax
487 mov 8*0($a_ptr), $acc1
488 mov 8*1($a_ptr), $acc2
489 mov 8*2($a_ptr), $acc3
490 mov 8*3($a_ptr), $acc4
492 call __ecp_nistz256_mul_montq
494 $code.=<<___ if ($addx);
500 mov 8*0($b_org), %rdx
501 mov 8*0($a_ptr), $acc1
502 mov 8*1($a_ptr), $acc2
503 mov 8*2($a_ptr), $acc3
504 mov 8*3($a_ptr), $acc4
505 lea -128($a_ptr), $a_ptr # control u-op density
507 call __ecp_nistz256_mul_montx
518 .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
520 .type __ecp_nistz256_mul_montq,\@abi-omnipotent
522 __ecp_nistz256_mul_montq:
523 ########################################################################
527 mov .Lpoly+8*1(%rip),$poly1
533 mov .Lpoly+8*3(%rip),$poly3
552 ########################################################################
553 # First reduction step
554 # Basically now we want to multiply acc[0] by p256,
555 # and add the result to the acc.
556 # Due to the special form of p256 we do some optimizations
558 # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
559 # then we add acc[0] and get acc[0] x 2^96
565 add $acc0, $acc1 # +=acc[0]<<96
568 mov 8*1($b_ptr), %rax
573 ########################################################################
606 ########################################################################
607 # Second reduction step
615 mov 8*2($b_ptr), %rax
620 ########################################################################
653 ########################################################################
654 # Third reduction step
662 mov 8*3($b_ptr), %rax
667 ########################################################################
700 ########################################################################
701 # Final reduction step
714 ########################################################################
715 # Branch-less conditional subtraction of P
716 sub \$-1, $acc4 # .Lpoly[0]
718 sbb $poly1, $acc5 # .Lpoly[1]
719 sbb \$0, $acc0 # .Lpoly[2]
721 sbb $poly3, $acc1 # .Lpoly[3]
726 mov $acc4, 8*0($r_ptr)
728 mov $acc5, 8*1($r_ptr)
730 mov $acc0, 8*2($r_ptr)
731 mov $acc1, 8*3($r_ptr)
734 .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq
736 ################################################################################
737 # void ecp_nistz256_sqr_mont(
741 # we optimize the square according to S.Gueron and V.Krasnov,
742 # "Speeding up Big-Number Squaring"
743 .globl ecp_nistz256_sqr_mont
744 .type ecp_nistz256_sqr_mont,\@function,2
746 ecp_nistz256_sqr_mont:
748 $code.=<<___ if ($addx);
750 and OPENSSL_ia32cap_P+8(%rip), %ecx
760 $code.=<<___ if ($addx);
765 mov 8*0($a_ptr), %rax
766 mov 8*1($a_ptr), $acc6
767 mov 8*2($a_ptr), $acc7
768 mov 8*3($a_ptr), $acc0
770 call __ecp_nistz256_sqr_montq
772 $code.=<<___ if ($addx);
777 mov 8*0($a_ptr), %rdx
778 mov 8*1($a_ptr), $acc6
779 mov 8*2($a_ptr), $acc7
780 mov 8*3($a_ptr), $acc0
781 lea -128($a_ptr), $a_ptr # control u-op density
783 call __ecp_nistz256_sqr_montx
794 .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
796 .type __ecp_nistz256_sqr_montq,\@abi-omnipotent
798 __ecp_nistz256_sqr_montq:
800 mulq $acc6 # a[1]*a[0]
805 mulq $acc5 # a[0]*a[2]
811 mulq $acc5 # a[0]*a[3]
817 #################################
818 mulq $acc6 # a[1]*a[2]
824 mulq $acc6 # a[1]*a[3]
832 #################################
833 mulq $acc7 # a[2]*a[3]
836 mov 8*0($a_ptr), %rax
840 add $acc1, $acc1 # acc1:6<<1
850 mov 8*1($a_ptr), %rax
856 mov 8*2($a_ptr), %rax
863 mov 8*3($a_ptr), %rax
873 mov .Lpoly+8*1(%rip), $a_ptr
874 mov .Lpoly+8*3(%rip), $t1
876 ##########################################
883 add $acc0, $acc1 # +=acc[0]<<96
889 ##########################################
902 ##########################################
915 ###########################################
928 ############################################
929 # Add the rest of the acc
938 sub \$-1, $acc4 # .Lpoly[0]
940 sbb $a_ptr, $acc5 # .Lpoly[1]
941 sbb \$0, $acc6 # .Lpoly[2]
943 sbb $t1, $acc7 # .Lpoly[3]
948 mov $acc4, 8*0($r_ptr)
950 mov $acc5, 8*1($r_ptr)
952 mov $acc6, 8*2($r_ptr)
953 mov $acc7, 8*3($r_ptr)
956 .size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq
961 .type __ecp_nistz256_mul_montx,\@abi-omnipotent
963 __ecp_nistz256_mul_montx:
964 ########################################################################
966 mulx $acc1, $acc0, $acc1
967 mulx $acc2, $t0, $acc2
969 xor $acc5, $acc5 # cf=0
970 mulx $acc3, $t1, $acc3
971 mov .Lpoly+8*3(%rip), $poly3
973 mulx $acc4, $t0, $acc4
976 shlx $poly1,$acc0,$t1
978 shrx $poly1,$acc0,$t0
981 ########################################################################
982 # First reduction step
986 mulx $poly3, $t0, $t1
987 mov 8*1($b_ptr), %rdx
991 xor $acc0, $acc0 # $acc0=0,cf=0,of=0
993 ########################################################################
995 mulx 8*0+128($a_ptr), $t0, $t1
999 mulx 8*1+128($a_ptr), $t0, $t1
1003 mulx 8*2+128($a_ptr), $t0, $t1
1007 mulx 8*3+128($a_ptr), $t0, $t1
1010 shlx $poly1, $acc1, $t0
1012 shrx $poly1, $acc1, $t1
1018 ########################################################################
1019 # Second reduction step
1023 mulx $poly3, $t0, $t1
1024 mov 8*2($b_ptr), %rdx
1028 xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
1030 ########################################################################
1032 mulx 8*0+128($a_ptr), $t0, $t1
1036 mulx 8*1+128($a_ptr), $t0, $t1
1040 mulx 8*2+128($a_ptr), $t0, $t1
1044 mulx 8*3+128($a_ptr), $t0, $t1
1047 shlx $poly1, $acc2, $t0
1049 shrx $poly1, $acc2, $t1
1055 ########################################################################
1056 # Third reduction step
1060 mulx $poly3, $t0, $t1
1061 mov 8*3($b_ptr), %rdx
1065 xor $acc2, $acc2 # $acc2=0,cf=0,of=0
1067 ########################################################################
1069 mulx 8*0+128($a_ptr), $t0, $t1
1073 mulx 8*1+128($a_ptr), $t0, $t1
1077 mulx 8*2+128($a_ptr), $t0, $t1
1081 mulx 8*3+128($a_ptr), $t0, $t1
1084 shlx $poly1, $acc3, $t0
1086 shrx $poly1, $acc3, $t1
1092 ########################################################################
1093 # Fourth reduction step
1097 mulx $poly3, $t0, $t1
1099 mov .Lpoly+8*1(%rip), $poly1
1105 ########################################################################
1106 # Branch-less conditional subtraction of P
1109 sbb \$-1, $acc4 # .Lpoly[0]
1110 sbb $poly1, $acc5 # .Lpoly[1]
1111 sbb \$0, $acc0 # .Lpoly[2]
1113 sbb $poly3, $acc1 # .Lpoly[3]
1118 mov $acc4, 8*0($r_ptr)
1120 mov $acc5, 8*1($r_ptr)
1122 mov $acc0, 8*2($r_ptr)
1123 mov $acc1, 8*3($r_ptr)
1126 .size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx
1128 .type __ecp_nistz256_sqr_montx,\@abi-omnipotent
1130 __ecp_nistz256_sqr_montx:
1131 mulx $acc6, $acc1, $acc2 # a[0]*a[1]
1132 mulx $acc7, $t0, $acc3 # a[0]*a[2]
1135 mulx $acc0, $t1, $acc4 # a[0]*a[3]
1139 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
1141 #################################
1142 mulx $acc7, $t0, $t1 # a[1]*a[2]
1146 mulx $acc0, $t0, $t1 # a[1]*a[3]
1152 #################################
1153 mulx $acc0, $t0, $acc6 # a[2]*a[3]
1154 mov 8*0+128($a_ptr), %rdx
1155 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
1156 adcx $acc1, $acc1 # acc1:6<<1
1159 adox $acc7, $acc6 # of=0
1161 mulx %rdx, $acc0, $t1
1162 mov 8*1+128($a_ptr), %rdx
1167 mov 8*2+128($a_ptr), %rdx
1173 mov 8*3+128($a_ptr), %rdx
1183 shlx $a_ptr, $acc0, $t0
1185 shrx $a_ptr, $acc0, $t4
1186 mov .Lpoly+8*3(%rip), $t1
1192 mulx $t1, $t0, $acc0
1195 shlx $a_ptr, $acc1, $t0
1197 shrx $a_ptr, $acc1, $t4
1203 mulx $t1, $t0, $acc1
1206 shlx $a_ptr, $acc2, $t0
1208 shrx $a_ptr, $acc2, $t4
1214 mulx $t1, $t0, $acc2
1217 shlx $a_ptr, $acc3, $t0
1219 shrx $a_ptr, $acc3, $t4
1225 mulx $t1, $t0, $acc3
1230 adc $acc0, $acc4 # accumulate upper half
1231 mov .Lpoly+8*1(%rip), $a_ptr
1239 xor %eax, %eax # cf=0
1240 sbb \$-1, $acc4 # .Lpoly[0]
1242 sbb $a_ptr, $acc5 # .Lpoly[1]
1243 sbb \$0, $acc6 # .Lpoly[2]
1245 sbb $t1, $acc7 # .Lpoly[3]
1250 mov $acc4, 8*0($r_ptr)
1252 mov $acc5, 8*1($r_ptr)
1254 mov $acc6, 8*2($r_ptr)
1255 mov $acc7, 8*3($r_ptr)
1258 .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx
1263 my ($r_ptr,$in_ptr)=("%rdi","%rsi");
1264 my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
1265 my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
1268 ################################################################################
1269 # void ecp_nistz256_from_mont(
1272 # This one performs Montgomery multiplication by 1, so we only need the reduction
1274 .globl ecp_nistz256_from_mont
1275 .type ecp_nistz256_from_mont,\@function,2
1277 ecp_nistz256_from_mont:
1281 mov 8*0($in_ptr), %rax
1282 mov .Lpoly+8*3(%rip), $t2
1283 mov 8*1($in_ptr), $acc1
1284 mov 8*2($in_ptr), $acc2
1285 mov 8*3($in_ptr), $acc3
1287 mov .Lpoly+8*1(%rip), $t1
1289 #########################################
1301 #########################################
1314 ##########################################
1327 ###########################################
1341 ###########################################
1342 # Branch-less conditional subtraction
1352 cmovnz $in_ptr, $acc1
1353 mov $acc0, 8*0($r_ptr)
1355 mov $acc1, 8*1($r_ptr)
1357 mov $acc2, 8*2($r_ptr)
1358 mov $acc3, 8*3($r_ptr)
1363 .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
1367 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1368 my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
1369 my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
1370 my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
1373 ################################################################################
1374 # void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index);
1375 .globl ecp_nistz256_select_w5
1376 .type ecp_nistz256_select_w5,\@abi-omnipotent
1378 ecp_nistz256_select_w5:
1380 $code.=<<___ if ($avx>1);
1381 mov OPENSSL_ia32cap_P+8(%rip), %eax
1383 jnz .Lavx2_select_w5
1385 $code.=<<___ if ($win64);
1386 lea -0x88(%rsp), %rax
1387 .LSEH_begin_ecp_nistz256_select_w5:
1388 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1389 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1390 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1391 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1392 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1393 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1394 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1395 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1396 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1397 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1398 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1401 movdqa .LOne(%rip), $ONE
1412 pshufd \$0, $INDEX, $INDEX
1415 .Lselect_loop_sse_w5:
1419 pcmpeqd $INDEX, $TMP0
1421 movdqa 16*0($in_t), $T0a
1422 movdqa 16*1($in_t), $T0b
1423 movdqa 16*2($in_t), $T0c
1424 movdqa 16*3($in_t), $T0d
1425 movdqa 16*4($in_t), $T0e
1426 movdqa 16*5($in_t), $T0f
1427 lea 16*6($in_t), $in_t
1443 jnz .Lselect_loop_sse_w5
1445 movdqu $Ra, 16*0($val)
1446 movdqu $Rb, 16*1($val)
1447 movdqu $Rc, 16*2($val)
1448 movdqu $Rd, 16*3($val)
1449 movdqu $Re, 16*4($val)
1450 movdqu $Rf, 16*5($val)
1452 $code.=<<___ if ($win64);
1453 movaps (%rsp), %xmm6
1454 movaps 0x10(%rsp), %xmm7
1455 movaps 0x20(%rsp), %xmm8
1456 movaps 0x30(%rsp), %xmm9
1457 movaps 0x40(%rsp), %xmm10
1458 movaps 0x50(%rsp), %xmm11
1459 movaps 0x60(%rsp), %xmm12
1460 movaps 0x70(%rsp), %xmm13
1461 movaps 0x80(%rsp), %xmm14
1462 movaps 0x90(%rsp), %xmm15
1463 lea 0xa8(%rsp), %rsp
1464 .LSEH_end_ecp_nistz256_select_w5:
1468 .size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5
1470 ################################################################################
1471 # void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index);
1472 .globl ecp_nistz256_select_w7
1473 .type ecp_nistz256_select_w7,\@abi-omnipotent
1475 ecp_nistz256_select_w7:
1477 $code.=<<___ if ($avx>1);
1478 mov OPENSSL_ia32cap_P+8(%rip), %eax
1480 jnz .Lavx2_select_w7
1482 $code.=<<___ if ($win64);
1483 lea -0x88(%rsp), %rax
1484 .LSEH_begin_ecp_nistz256_select_w7:
1485 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1486 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1487 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1488 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1489 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1490 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1491 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1492 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1493 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1494 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1495 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1498 movdqa .LOne(%rip), $M0
1507 pshufd \$0, $INDEX, $INDEX
1510 .Lselect_loop_sse_w7:
1513 movdqa 16*0($in_t), $T0a
1514 movdqa 16*1($in_t), $T0b
1515 pcmpeqd $INDEX, $TMP0
1516 movdqa 16*2($in_t), $T0c
1517 movdqa 16*3($in_t), $T0d
1518 lea 16*4($in_t), $in_t
1527 prefetcht0 255($in_t)
1531 jnz .Lselect_loop_sse_w7
1533 movdqu $Ra, 16*0($val)
1534 movdqu $Rb, 16*1($val)
1535 movdqu $Rc, 16*2($val)
1536 movdqu $Rd, 16*3($val)
1538 $code.=<<___ if ($win64);
1539 movaps (%rsp), %xmm6
1540 movaps 0x10(%rsp), %xmm7
1541 movaps 0x20(%rsp), %xmm8
1542 movaps 0x30(%rsp), %xmm9
1543 movaps 0x40(%rsp), %xmm10
1544 movaps 0x50(%rsp), %xmm11
1545 movaps 0x60(%rsp), %xmm12
1546 movaps 0x70(%rsp), %xmm13
1547 movaps 0x80(%rsp), %xmm14
1548 movaps 0x90(%rsp), %xmm15
1549 lea 0xa8(%rsp), %rsp
1550 .LSEH_end_ecp_nistz256_select_w7:
1554 .size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7
1558 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1559 my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
1560 my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
1561 my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
1564 ################################################################################
1565 # void ecp_nistz256_avx2_select_w5(uint64_t *val, uint64_t *in_t, int index);
1566 .type ecp_nistz256_avx2_select_w5,\@abi-omnipotent
1568 ecp_nistz256_avx2_select_w5:
1572 $code.=<<___ if ($win64);
1573 lea -0x88(%rsp), %rax
1574 .LSEH_begin_ecp_nistz256_avx2_select_w5:
1575 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1576 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1577 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1578 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1579 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1580 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1581 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1582 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1583 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1584 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1585 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1588 vmovdqa .LTwo(%rip), $TWO
1594 vmovdqa .LOne(%rip), $M0
1595 vmovdqa .LTwo(%rip), $M1
1598 vpermd $INDEX, $Ra, $INDEX
1601 .Lselect_loop_avx2_w5:
1603 vmovdqa 32*0($in_t), $T0a
1604 vmovdqa 32*1($in_t), $T0b
1605 vmovdqa 32*2($in_t), $T0c
1607 vmovdqa 32*3($in_t), $T1a
1608 vmovdqa 32*4($in_t), $T1b
1609 vmovdqa 32*5($in_t), $T1c
1611 vpcmpeqd $INDEX, $M0, $TMP0
1612 vpcmpeqd $INDEX, $M1, $TMP1
1614 vpaddd $TWO, $M0, $M0
1615 vpaddd $TWO, $M1, $M1
1616 lea 32*6($in_t), $in_t
1618 vpand $TMP0, $T0a, $T0a
1619 vpand $TMP0, $T0b, $T0b
1620 vpand $TMP0, $T0c, $T0c
1621 vpand $TMP1, $T1a, $T1a
1622 vpand $TMP1, $T1b, $T1b
1623 vpand $TMP1, $T1c, $T1c
1625 vpxor $T0a, $Ra, $Ra
1626 vpxor $T0b, $Rb, $Rb
1627 vpxor $T0c, $Rc, $Rc
1628 vpxor $T1a, $Ra, $Ra
1629 vpxor $T1b, $Rb, $Rb
1630 vpxor $T1c, $Rc, $Rc
1633 jnz .Lselect_loop_avx2_w5
1635 vmovdqu $Ra, 32*0($val)
1636 vmovdqu $Rb, 32*1($val)
1637 vmovdqu $Rc, 32*2($val)
1640 $code.=<<___ if ($win64);
1641 movaps (%rsp), %xmm6
1642 movaps 0x10(%rsp), %xmm7
1643 movaps 0x20(%rsp), %xmm8
1644 movaps 0x30(%rsp), %xmm9
1645 movaps 0x40(%rsp), %xmm10
1646 movaps 0x50(%rsp), %xmm11
1647 movaps 0x60(%rsp), %xmm12
1648 movaps 0x70(%rsp), %xmm13
1649 movaps 0x80(%rsp), %xmm14
1650 movaps 0x90(%rsp), %xmm15
1651 lea 0xa8(%rsp), %rsp
1652 .LSEH_end_ecp_nistz256_avx2_select_w5:
1656 .size ecp_nistz256_avx2_select_w5,.-ecp_nistz256_avx2_select_w5
1660 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1661 my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
1662 my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
1663 my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
1664 my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
1668 ################################################################################
1669 # void ecp_nistz256_avx2_select_w7(uint64_t *val, uint64_t *in_t, int index);
1670 .globl ecp_nistz256_avx2_select_w7
1671 .type ecp_nistz256_avx2_select_w7,\@abi-omnipotent
1673 ecp_nistz256_avx2_select_w7:
1677 $code.=<<___ if ($win64);
1678 lea -0x88(%rsp), %rax
1679 .LSEH_begin_ecp_nistz256_avx2_select_w7:
1680 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1681 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1682 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1683 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1684 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1685 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1686 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1687 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1688 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1689 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1690 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1693 vmovdqa .LThree(%rip), $THREE
1698 vmovdqa .LOne(%rip), $M0
1699 vmovdqa .LTwo(%rip), $M1
1700 vmovdqa .LThree(%rip), $M2
1703 vpermd $INDEX, $Ra, $INDEX
1704 # Skip index = 0, because it is implicitly the point at infinity
1707 .Lselect_loop_avx2_w7:
1709 vmovdqa 32*0($in_t), $T0a
1710 vmovdqa 32*1($in_t), $T0b
1712 vmovdqa 32*2($in_t), $T1a
1713 vmovdqa 32*3($in_t), $T1b
1715 vmovdqa 32*4($in_t), $T2a
1716 vmovdqa 32*5($in_t), $T2b
1718 vpcmpeqd $INDEX, $M0, $TMP0
1719 vpcmpeqd $INDEX, $M1, $TMP1
1720 vpcmpeqd $INDEX, $M2, $TMP2
1722 vpaddd $THREE, $M0, $M0
1723 vpaddd $THREE, $M1, $M1
1724 vpaddd $THREE, $M2, $M2
1725 lea 32*6($in_t), $in_t
1727 vpand $TMP0, $T0a, $T0a
1728 vpand $TMP0, $T0b, $T0b
1729 vpand $TMP1, $T1a, $T1a
1730 vpand $TMP1, $T1b, $T1b
1731 vpand $TMP2, $T2a, $T2a
1732 vpand $TMP2, $T2b, $T2b
1734 vpxor $T0a, $Ra, $Ra
1735 vpxor $T0b, $Rb, $Rb
1736 vpxor $T1a, $Ra, $Ra
1737 vpxor $T1b, $Rb, $Rb
1738 vpxor $T2a, $Ra, $Ra
1739 vpxor $T2b, $Rb, $Rb
1742 jnz .Lselect_loop_avx2_w7
1745 vmovdqa 32*0($in_t), $T0a
1746 vmovdqa 32*1($in_t), $T0b
1748 vpcmpeqd $INDEX, $M0, $TMP0
1750 vpand $TMP0, $T0a, $T0a
1751 vpand $TMP0, $T0b, $T0b
1753 vpxor $T0a, $Ra, $Ra
1754 vpxor $T0b, $Rb, $Rb
1756 vmovdqu $Ra, 32*0($val)
1757 vmovdqu $Rb, 32*1($val)
1760 $code.=<<___ if ($win64);
1761 movaps (%rsp), %xmm6
1762 movaps 0x10(%rsp), %xmm7
1763 movaps 0x20(%rsp), %xmm8
1764 movaps 0x30(%rsp), %xmm9
1765 movaps 0x40(%rsp), %xmm10
1766 movaps 0x50(%rsp), %xmm11
1767 movaps 0x60(%rsp), %xmm12
1768 movaps 0x70(%rsp), %xmm13
1769 movaps 0x80(%rsp), %xmm14
1770 movaps 0x90(%rsp), %xmm15
1771 lea 0xa8(%rsp), %rsp
1772 .LSEH_end_ecp_nistz256_avx2_select_w7:
1776 .size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7
1780 .globl ecp_nistz256_avx2_select_w7
1781 .type ecp_nistz256_avx2_select_w7,\@function,3
1783 ecp_nistz256_avx2_select_w7:
1784 .byte 0x0f,0x0b # ud2
1786 .size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7
1790 ########################################################################
1791 # This block implements higher level point_double, point_add and
1792 # point_add_affine. The key to performance in this case is to allow
1793 # out-of-order execution logic to overlap computations from next step
1794 # with tail processing from current step. By using tailored calling
1795 # sequence we minimize inter-step overhead to give processor better
1796 # shot at overlapping operations...
1798 # You will notice that input data is copied to stack. Trouble is that
1799 # there are no registers to spare for holding original pointers and
1800 # reloading them, pointers, would create undesired dependencies on
1801 # effective addresses calculation paths. In other words it's too done
1802 # to favour out-of-order execution logic.
1803 # <appro@openssl.org>
1805 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
1806 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
1807 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
1808 my ($poly1,$poly3)=($acc6,$acc7);
1810 sub load_for_mul () {
1811 my ($a,$b,$src0) = @_;
1812 my $bias = $src0 eq "%rax" ? 0 : -128;
1818 lea $bias+$a, $a_ptr
1823 sub load_for_sqr () {
1825 my $bias = $src0 eq "%rax" ? 0 : -128;
1829 lea $bias+$a, $a_ptr
1835 ########################################################################
1836 # operate in 4-5-0-1 "name space" that matches multiplication output
1838 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
1841 .type __ecp_nistz256_add_toq,\@abi-omnipotent
1843 __ecp_nistz256_add_toq:
1845 add 8*0($b_ptr), $a0
1846 adc 8*1($b_ptr), $a1
1848 adc 8*2($b_ptr), $a2
1849 adc 8*3($b_ptr), $a3
1863 mov $a0, 8*0($r_ptr)
1865 mov $a1, 8*1($r_ptr)
1867 mov $a2, 8*2($r_ptr)
1868 mov $a3, 8*3($r_ptr)
1871 .size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq
1873 .type __ecp_nistz256_sub_fromq,\@abi-omnipotent
1875 __ecp_nistz256_sub_fromq:
1876 sub 8*0($b_ptr), $a0
1877 sbb 8*1($b_ptr), $a1
1879 sbb 8*2($b_ptr), $a2
1880 sbb 8*3($b_ptr), $a3
1894 mov $a0, 8*0($r_ptr)
1896 mov $a1, 8*1($r_ptr)
1898 mov $a2, 8*2($r_ptr)
1899 mov $a3, 8*3($r_ptr)
1902 .size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq
1904 .type __ecp_nistz256_subq,\@abi-omnipotent
1906 __ecp_nistz256_subq:
1929 .size __ecp_nistz256_subq,.-__ecp_nistz256_subq
1931 .type __ecp_nistz256_mul_by_2q,\@abi-omnipotent
1933 __ecp_nistz256_mul_by_2q:
1935 add $a0, $a0 # a0:a3+a0:a3
1953 mov $a0, 8*0($r_ptr)
1955 mov $a1, 8*1($r_ptr)
1957 mov $a2, 8*2($r_ptr)
1958 mov $a3, 8*3($r_ptr)
1961 .size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q
1966 my ($src0,$sfx,$bias);
1967 my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
1975 .globl ecp_nistz256_point_double
1976 .type ecp_nistz256_point_double,\@function,2
1978 ecp_nistz256_point_double:
1980 $code.=<<___ if ($addx);
1982 and OPENSSL_ia32cap_P+8(%rip), %ecx
1992 .type ecp_nistz256_point_doublex,\@function,2
1994 ecp_nistz256_point_doublex:
2007 .Lpoint_double_shortcut$x:
2008 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
2009 mov $a_ptr, $b_ptr # backup copy
2010 movdqu 0x10($a_ptr), %xmm1
2011 mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
2012 mov 0x20+8*1($a_ptr), $acc5
2013 mov 0x20+8*2($a_ptr), $acc0
2014 mov 0x20+8*3($a_ptr), $acc1
2015 mov .Lpoly+8*1(%rip), $poly1
2016 mov .Lpoly+8*3(%rip), $poly3
2017 movdqa %xmm0, $in_x(%rsp)
2018 movdqa %xmm1, $in_x+0x10(%rsp)
2019 lea 0x20($r_ptr), $acc2
2020 lea 0x40($r_ptr), $acc3
2025 lea $S(%rsp), $r_ptr
2026 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y);
2028 mov 0x40+8*0($a_ptr), $src0
2029 mov 0x40+8*1($a_ptr), $acc6
2030 mov 0x40+8*2($a_ptr), $acc7
2031 mov 0x40+8*3($a_ptr), $acc0
2032 lea 0x40-$bias($a_ptr), $a_ptr
2033 lea $Zsqr(%rsp), $r_ptr
2034 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z);
2036 `&load_for_sqr("$S(%rsp)", "$src0")`
2037 lea $S(%rsp), $r_ptr
2038 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S);
2040 mov 0x20($b_ptr), $src0 # $b_ptr is still valid
2041 mov 0x40+8*0($b_ptr), $acc1
2042 mov 0x40+8*1($b_ptr), $acc2
2043 mov 0x40+8*2($b_ptr), $acc3
2044 mov 0x40+8*3($b_ptr), $acc4
2045 lea 0x40-$bias($b_ptr), $a_ptr
2046 lea 0x20($b_ptr), $b_ptr
2048 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y);
2049 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z);
2051 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2052 mov $in_x+8*1(%rsp), $acc5
2053 lea $Zsqr(%rsp), $b_ptr
2054 mov $in_x+8*2(%rsp), $acc0
2055 mov $in_x+8*3(%rsp), $acc1
2056 lea $M(%rsp), $r_ptr
2057 call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr);
2059 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2060 mov $in_x+8*1(%rsp), $acc5
2061 lea $Zsqr(%rsp), $b_ptr
2062 mov $in_x+8*2(%rsp), $acc0
2063 mov $in_x+8*3(%rsp), $acc1
2064 lea $Zsqr(%rsp), $r_ptr
2065 call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr);
2067 `&load_for_sqr("$S(%rsp)", "$src0")`
2069 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S);
2072 ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
2073 # operate in 4-5-6-7 "name space" that matches squaring output
2075 my ($poly1,$poly3)=($a_ptr,$t1);
2076 my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
2089 xor $a_ptr, $a_ptr # borrow $a_ptr
2098 mov $a1, $t0 # a0:a3>>1
2109 mov $a0, 8*0($r_ptr)
2111 mov $a1, 8*1($r_ptr)
2115 mov $a2, 8*2($r_ptr)
2116 mov $a3, 8*3($r_ptr)
2120 `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
2121 lea $M(%rsp), $r_ptr
2122 call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr);
2124 lea $tmp0(%rsp), $r_ptr
2125 call __ecp_nistz256_mul_by_2$x
2127 lea $M(%rsp), $b_ptr
2128 lea $M(%rsp), $r_ptr
2129 call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M);
2131 `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
2132 lea $S(%rsp), $r_ptr
2133 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x);
2135 lea $tmp0(%rsp), $r_ptr
2136 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S);
2138 `&load_for_sqr("$M(%rsp)", "$src0")`
2140 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M);
2142 lea $tmp0(%rsp), $b_ptr
2143 mov $acc6, $acc0 # harmonize sqr output and sub input
2147 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0);
2149 mov $S+8*0(%rsp), $t0
2150 mov $S+8*1(%rsp), $t1
2151 mov $S+8*2(%rsp), $t2
2152 mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order
2153 lea $S(%rsp), $r_ptr
2154 call __ecp_nistz256_sub$x # p256_sub(S, S, res_x);
2157 lea $M(%rsp), $b_ptr
2158 mov $acc4, $acc6 # harmonize sub output and mul input
2160 mov $acc4, $S+8*0(%rsp) # have to save:-(
2162 mov $acc5, $S+8*1(%rsp)
2164 mov $acc0, $S+8*2(%rsp)
2165 lea $S-$bias(%rsp), $a_ptr
2167 mov $acc1, $S+8*3(%rsp)
2169 lea $S(%rsp), $r_ptr
2170 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M);
2174 call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y);
2184 .size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx
2191 my ($src0,$sfx,$bias);
2192 my ($H,$Hsqr,$R,$Rsqr,$Hcub,
2194 $res_x,$res_y,$res_z,
2195 $in1_x,$in1_y,$in1_z,
2196 $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
2197 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
2205 .globl ecp_nistz256_point_add
2206 .type ecp_nistz256_point_add,\@function,3
2208 ecp_nistz256_point_add:
2210 $code.=<<___ if ($addx);
2212 and OPENSSL_ia32cap_P+8(%rip), %ecx
2222 .type ecp_nistz256_point_addx,\@function,3
2224 ecp_nistz256_point_addx:
2237 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2238 movdqu 0x10($a_ptr), %xmm1
2239 movdqu 0x20($a_ptr), %xmm2
2240 movdqu 0x30($a_ptr), %xmm3
2241 movdqu 0x40($a_ptr), %xmm4
2242 movdqu 0x50($a_ptr), %xmm5
2243 mov $a_ptr, $b_ptr # reassign
2244 mov $b_org, $a_ptr # reassign
2245 movdqa %xmm0, $in1_x(%rsp)
2246 movdqa %xmm1, $in1_x+0x10(%rsp)
2247 movdqa %xmm2, $in1_y(%rsp)
2248 movdqa %xmm3, $in1_y+0x10(%rsp)
2249 movdqa %xmm4, $in1_z(%rsp)
2250 movdqa %xmm5, $in1_z+0x10(%rsp)
2253 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
2254 pshufd \$0xb1, %xmm5, %xmm3
2255 movdqu 0x10($a_ptr), %xmm1
2256 movdqu 0x20($a_ptr), %xmm2
2258 movdqu 0x30($a_ptr), %xmm3
2259 mov 0x40+8*0($a_ptr), $src0 # load original in2_z
2260 mov 0x40+8*1($a_ptr), $acc6
2261 mov 0x40+8*2($a_ptr), $acc7
2262 mov 0x40+8*3($a_ptr), $acc0
2263 movdqa %xmm0, $in2_x(%rsp)
2264 pshufd \$0x1e, %xmm5, %xmm4
2265 movdqa %xmm1, $in2_x+0x10(%rsp)
2266 movdqu 0x40($a_ptr),%xmm0 # in2_z again
2267 movdqu 0x50($a_ptr),%xmm1
2268 movdqa %xmm2, $in2_y(%rsp)
2269 movdqa %xmm3, $in2_y+0x10(%rsp)
2273 movq $r_ptr, %xmm0 # save $r_ptr
2275 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2276 mov $src0, $in2_z+8*0(%rsp) # make in2_z copy
2277 mov $acc6, $in2_z+8*1(%rsp)
2278 mov $acc7, $in2_z+8*2(%rsp)
2279 mov $acc0, $in2_z+8*3(%rsp)
2280 lea $Z2sqr(%rsp), $r_ptr # Z2^2
2281 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z);
2283 pcmpeqd %xmm4, %xmm5
2284 pshufd \$0xb1, %xmm1, %xmm4
2286 pshufd \$0, %xmm5, %xmm5 # in1infty
2287 pshufd \$0x1e, %xmm4, %xmm3
2290 pcmpeqd %xmm3, %xmm4
2291 pshufd \$0, %xmm4, %xmm4 # in2infty
2292 mov 0x40+8*0($b_ptr), $src0 # load original in1_z
2293 mov 0x40+8*1($b_ptr), $acc6
2294 mov 0x40+8*2($b_ptr), $acc7
2295 mov 0x40+8*3($b_ptr), $acc0
2298 lea 0x40-$bias($b_ptr), $a_ptr
2299 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2300 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2302 `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
2303 lea $S1(%rsp), $r_ptr # S1 = Z2^3
2304 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z);
2306 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2307 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2308 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2310 `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
2311 lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3
2312 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y);
2314 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2315 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2316 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2318 lea $S1(%rsp), $b_ptr
2319 lea $R(%rsp), $r_ptr # R = S2 - S1
2320 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1);
2322 or $acc5, $acc4 # see if result is zero
2326 por %xmm5, %xmm2 # in1infty || in2infty
2329 `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2330 lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2
2331 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr);
2333 `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
2334 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2335 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr);
2337 lea $U1(%rsp), $b_ptr
2338 lea $H(%rsp), $r_ptr # H = U2 - U1
2339 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1);
2341 or $acc5, $acc4 # see if result is zero
2345 .byte 0x3e # predict taken
2346 jnz .Ladd_proceed$x # is_equal(U1,U2)?
2350 jnz .Ladd_proceed$x # (in1infty || in2infty)?
2352 jz .Ladd_double$x # is_equal(S1,S2)?
2354 movq %xmm0, $r_ptr # restore $r_ptr
2356 movdqu %xmm0, 0x00($r_ptr)
2357 movdqu %xmm0, 0x10($r_ptr)
2358 movdqu %xmm0, 0x20($r_ptr)
2359 movdqu %xmm0, 0x30($r_ptr)
2360 movdqu %xmm0, 0x40($r_ptr)
2361 movdqu %xmm0, 0x50($r_ptr)
2366 movq %xmm1, $a_ptr # restore $a_ptr
2367 movq %xmm0, $r_ptr # restore $r_ptr
2368 add \$`32*(18-5)`, %rsp # difference in frame sizes
2369 jmp .Lpoint_double_shortcut$x
2373 `&load_for_sqr("$R(%rsp)", "$src0")`
2374 lea $Rsqr(%rsp), $r_ptr # R^2
2375 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2377 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2378 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2379 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2381 `&load_for_sqr("$H(%rsp)", "$src0")`
2382 lea $Hsqr(%rsp), $r_ptr # H^2
2383 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2385 `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
2386 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2387 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z);
2389 `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
2390 lea $Hcub(%rsp), $r_ptr # H^3
2391 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2393 `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
2394 lea $U2(%rsp), $r_ptr # U1*H^2
2395 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr);
2398 #######################################################################
2399 # operate in 4-5-0-1 "name space" that matches multiplication output
2401 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2402 my ($poly1, $poly3)=($acc6,$acc7);
2405 #lea $U2(%rsp), $a_ptr
2406 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2407 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2410 add $acc0, $acc0 # a0:a3+a0:a3
2411 lea $Rsqr(%rsp), $a_ptr
2428 mov 8*0($a_ptr), $t0
2430 mov 8*1($a_ptr), $t1
2432 mov 8*2($a_ptr), $t2
2434 mov 8*3($a_ptr), $t3
2436 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2438 lea $Hcub(%rsp), $b_ptr
2439 lea $res_x(%rsp), $r_ptr
2440 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2442 mov $U2+8*0(%rsp), $t0
2443 mov $U2+8*1(%rsp), $t1
2444 mov $U2+8*2(%rsp), $t2
2445 mov $U2+8*3(%rsp), $t3
2446 lea $res_y(%rsp), $r_ptr
2448 call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x);
2450 mov $acc0, 8*0($r_ptr) # save the result, as
2451 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2452 mov $acc2, 8*2($r_ptr)
2453 mov $acc3, 8*3($r_ptr)
2457 `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
2458 lea $S2(%rsp), $r_ptr
2459 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub);
2461 `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
2462 lea $res_y(%rsp), $r_ptr
2463 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y);
2465 lea $S2(%rsp), $b_ptr
2466 lea $res_y(%rsp), $r_ptr
2467 call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2);
2469 movq %xmm0, $r_ptr # restore $r_ptr
2471 movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
2473 pandn $res_z(%rsp), %xmm0
2475 pandn $res_z+0x10(%rsp), %xmm1
2477 pand $in2_z(%rsp), %xmm2
2478 pand $in2_z+0x10(%rsp), %xmm3
2482 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2488 pand $in1_z(%rsp), %xmm2
2489 pand $in1_z+0x10(%rsp), %xmm3
2492 movdqu %xmm2, 0x40($r_ptr)
2493 movdqu %xmm3, 0x50($r_ptr)
2495 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2497 pandn $res_x(%rsp), %xmm0
2499 pandn $res_x+0x10(%rsp), %xmm1
2501 pand $in2_x(%rsp), %xmm2
2502 pand $in2_x+0x10(%rsp), %xmm3
2506 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2512 pand $in1_x(%rsp), %xmm2
2513 pand $in1_x+0x10(%rsp), %xmm3
2516 movdqu %xmm2, 0x00($r_ptr)
2517 movdqu %xmm3, 0x10($r_ptr)
2519 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2521 pandn $res_y(%rsp), %xmm0
2523 pandn $res_y+0x10(%rsp), %xmm1
2525 pand $in2_y(%rsp), %xmm2
2526 pand $in2_y+0x10(%rsp), %xmm3
2530 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2536 pand $in1_y(%rsp), %xmm2
2537 pand $in1_y+0x10(%rsp), %xmm3
2540 movdqu %xmm2, 0x20($r_ptr)
2541 movdqu %xmm3, 0x30($r_ptr)
2552 .size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx
2557 sub gen_add_affine () {
2559 my ($src0,$sfx,$bias);
2560 my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
2561 $res_x,$res_y,$res_z,
2562 $in1_x,$in1_y,$in1_z,
2563 $in2_x,$in2_y)=map(32*$_,(0..14));
2572 .globl ecp_nistz256_point_add_affine
2573 .type ecp_nistz256_point_add_affine,\@function,3
2575 ecp_nistz256_point_add_affine:
2577 $code.=<<___ if ($addx);
2579 and OPENSSL_ia32cap_P+8(%rip), %ecx
2581 je .Lpoint_add_affinex
2589 .type ecp_nistz256_point_add_affinex,\@function,3
2591 ecp_nistz256_point_add_affinex:
2592 .Lpoint_add_affinex:
2604 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2605 mov $b_org, $b_ptr # reassign
2606 movdqu 0x10($a_ptr), %xmm1
2607 movdqu 0x20($a_ptr), %xmm2
2608 movdqu 0x30($a_ptr), %xmm3
2609 movdqu 0x40($a_ptr), %xmm4
2610 movdqu 0x50($a_ptr), %xmm5
2611 mov 0x40+8*0($a_ptr), $src0 # load original in1_z
2612 mov 0x40+8*1($a_ptr), $acc6
2613 mov 0x40+8*2($a_ptr), $acc7
2614 mov 0x40+8*3($a_ptr), $acc0
2615 movdqa %xmm0, $in1_x(%rsp)
2616 movdqa %xmm1, $in1_x+0x10(%rsp)
2617 movdqa %xmm2, $in1_y(%rsp)
2618 movdqa %xmm3, $in1_y+0x10(%rsp)
2619 movdqa %xmm4, $in1_z(%rsp)
2620 movdqa %xmm5, $in1_z+0x10(%rsp)
2623 movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
2624 pshufd \$0xb1, %xmm5, %xmm3
2625 movdqu 0x10($b_ptr), %xmm1
2626 movdqu 0x20($b_ptr), %xmm2
2628 movdqu 0x30($b_ptr), %xmm3
2629 movdqa %xmm0, $in2_x(%rsp)
2630 pshufd \$0x1e, %xmm5, %xmm4
2631 movdqa %xmm1, $in2_x+0x10(%rsp)
2633 movq $r_ptr, %xmm0 # save $r_ptr
2634 movdqa %xmm2, $in2_y(%rsp)
2635 movdqa %xmm3, $in2_y+0x10(%rsp)
2641 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2642 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2643 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2645 pcmpeqd %xmm4, %xmm5
2646 pshufd \$0xb1, %xmm3, %xmm4
2647 mov 0x00($b_ptr), $src0 # $b_ptr is still valid
2648 #lea 0x00($b_ptr), $b_ptr
2649 mov $acc4, $acc1 # harmonize sqr output and mul input
2651 pshufd \$0, %xmm5, %xmm5 # in1infty
2652 pshufd \$0x1e, %xmm4, %xmm3
2657 pcmpeqd %xmm3, %xmm4
2658 pshufd \$0, %xmm4, %xmm4 # in2infty
2660 lea $Z1sqr-$bias(%rsp), $a_ptr
2662 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2663 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x);
2665 lea $in1_x(%rsp), $b_ptr
2666 lea $H(%rsp), $r_ptr # H = U2 - U1
2667 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x);
2669 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2670 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2671 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2673 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2674 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2675 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2677 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2678 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2679 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2681 lea $in1_y(%rsp), $b_ptr
2682 lea $R(%rsp), $r_ptr # R = S2 - S1
2683 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y);
2685 `&load_for_sqr("$H(%rsp)", "$src0")`
2686 lea $Hsqr(%rsp), $r_ptr # H^2
2687 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2689 `&load_for_sqr("$R(%rsp)", "$src0")`
2690 lea $Rsqr(%rsp), $r_ptr # R^2
2691 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2693 `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
2694 lea $Hcub(%rsp), $r_ptr # H^3
2695 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2697 `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2698 lea $U2(%rsp), $r_ptr # U1*H^2
2699 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr);
2702 #######################################################################
2703 # operate in 4-5-0-1 "name space" that matches multiplication output
2705 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2706 my ($poly1, $poly3)=($acc6,$acc7);
2709 #lea $U2(%rsp), $a_ptr
2710 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2711 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2714 add $acc0, $acc0 # a0:a3+a0:a3
2715 lea $Rsqr(%rsp), $a_ptr
2732 mov 8*0($a_ptr), $t0
2734 mov 8*1($a_ptr), $t1
2736 mov 8*2($a_ptr), $t2
2738 mov 8*3($a_ptr), $t3
2740 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2742 lea $Hcub(%rsp), $b_ptr
2743 lea $res_x(%rsp), $r_ptr
2744 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2746 mov $U2+8*0(%rsp), $t0
2747 mov $U2+8*1(%rsp), $t1
2748 mov $U2+8*2(%rsp), $t2
2749 mov $U2+8*3(%rsp), $t3
2750 lea $H(%rsp), $r_ptr
2752 call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x);
2754 mov $acc0, 8*0($r_ptr) # save the result, as
2755 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2756 mov $acc2, 8*2($r_ptr)
2757 mov $acc3, 8*3($r_ptr)
2761 `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
2762 lea $S2(%rsp), $r_ptr
2763 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y);
2765 `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
2766 lea $H(%rsp), $r_ptr
2767 call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R);
2769 lea $S2(%rsp), $b_ptr
2770 lea $res_y(%rsp), $r_ptr
2771 call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2);
2773 movq %xmm0, $r_ptr # restore $r_ptr
2775 movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
2777 pandn $res_z(%rsp), %xmm0
2779 pandn $res_z+0x10(%rsp), %xmm1
2781 pand .LONE_mont(%rip), %xmm2
2782 pand .LONE_mont+0x10(%rip), %xmm3
2786 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2792 pand $in1_z(%rsp), %xmm2
2793 pand $in1_z+0x10(%rsp), %xmm3
2796 movdqu %xmm2, 0x40($r_ptr)
2797 movdqu %xmm3, 0x50($r_ptr)
2799 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2801 pandn $res_x(%rsp), %xmm0
2803 pandn $res_x+0x10(%rsp), %xmm1
2805 pand $in2_x(%rsp), %xmm2
2806 pand $in2_x+0x10(%rsp), %xmm3
2810 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2816 pand $in1_x(%rsp), %xmm2
2817 pand $in1_x+0x10(%rsp), %xmm3
2820 movdqu %xmm2, 0x00($r_ptr)
2821 movdqu %xmm3, 0x10($r_ptr)
2823 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2825 pandn $res_y(%rsp), %xmm0
2827 pandn $res_y+0x10(%rsp), %xmm1
2829 pand $in2_y(%rsp), %xmm2
2830 pand $in2_y+0x10(%rsp), %xmm3
2834 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2840 pand $in1_y(%rsp), %xmm2
2841 pand $in1_y+0x10(%rsp), %xmm3
2844 movdqu %xmm2, 0x20($r_ptr)
2845 movdqu %xmm3, 0x30($r_ptr)
2855 .size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx
2858 &gen_add_affine("q");
2860 ########################################################################
2864 ########################################################################
2865 # operate in 4-5-0-1 "name space" that matches multiplication output
2867 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2870 .type __ecp_nistz256_add_tox,\@abi-omnipotent
2872 __ecp_nistz256_add_tox:
2874 adc 8*0($b_ptr), $a0
2875 adc 8*1($b_ptr), $a1
2877 adc 8*2($b_ptr), $a2
2878 adc 8*3($b_ptr), $a3
2893 mov $a0, 8*0($r_ptr)
2895 mov $a1, 8*1($r_ptr)
2897 mov $a2, 8*2($r_ptr)
2898 mov $a3, 8*3($r_ptr)
2901 .size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox
2903 .type __ecp_nistz256_sub_fromx,\@abi-omnipotent
2905 __ecp_nistz256_sub_fromx:
2907 sbb 8*0($b_ptr), $a0
2908 sbb 8*1($b_ptr), $a1
2910 sbb 8*2($b_ptr), $a2
2911 sbb 8*3($b_ptr), $a3
2926 mov $a0, 8*0($r_ptr)
2928 mov $a1, 8*1($r_ptr)
2930 mov $a2, 8*2($r_ptr)
2931 mov $a3, 8*3($r_ptr)
2934 .size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx
2936 .type __ecp_nistz256_subx,\@abi-omnipotent
2938 __ecp_nistz256_subx:
2963 .size __ecp_nistz256_subx,.-__ecp_nistz256_subx
2965 .type __ecp_nistz256_mul_by_2x,\@abi-omnipotent
2967 __ecp_nistz256_mul_by_2x:
2969 adc $a0, $a0 # a0:a3+a0:a3
2988 mov $a0, 8*0($r_ptr)
2990 mov $a1, 8*1($r_ptr)
2992 mov $a2, 8*2($r_ptr)
2993 mov $a3, 8*3($r_ptr)
2996 .size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x
3001 &gen_add_affine("x");
3005 $code =~ s/\`([^\`]*)\`/eval $1/gem;