1 /*===---- avx512fintrin.h - AVX2 intrinsics --------------------------------===
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 *===-----------------------------------------------------------------------===
24 #error "Never use <avx512fintrin.h> directly; include <immintrin.h> instead."
27 #ifndef __AVX512FINTRIN_H
28 #define __AVX512FINTRIN_H
30 typedef double __v8df __attribute__((__vector_size__(64)));
31 typedef float __v16sf __attribute__((__vector_size__(64)));
32 typedef long long __v8di __attribute__((__vector_size__(64)));
33 typedef int __v16si __attribute__((__vector_size__(64)));
35 typedef float __m512 __attribute__((__vector_size__(64)));
36 typedef double __m512d __attribute__((__vector_size__(64)));
37 typedef long long __m512i __attribute__((__vector_size__(64)));
39 typedef unsigned char __mmask8;
40 typedef unsigned short __mmask16;
42 /* Rounding mode macros. */
43 #define _MM_FROUND_TO_NEAREST_INT 0x00
44 #define _MM_FROUND_TO_NEG_INF 0x01
45 #define _MM_FROUND_TO_POS_INF 0x02
46 #define _MM_FROUND_TO_ZERO 0x03
47 #define _MM_FROUND_CUR_DIRECTION 0x04
49 /* Define the default attributes for the functions in this file. */
50 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
52 /* Create vectors with repeated elements */
54 static __inline __m512i __DEFAULT_FN_ATTRS
55 _mm512_setzero_si512(void)
57 return (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 };
60 static __inline __m512i __DEFAULT_FN_ATTRS
61 _mm512_maskz_set1_epi32(__mmask16 __M, int __A)
63 return (__m512i) __builtin_ia32_pbroadcastd512_gpr_mask (__A,
65 _mm512_setzero_si512 (),
69 static __inline __m512i __DEFAULT_FN_ATTRS
70 _mm512_maskz_set1_epi64(__mmask8 __M, long long __A)
73 return (__m512i) __builtin_ia32_pbroadcastq512_gpr_mask (__A,
75 _mm512_setzero_si512 (),
78 return (__m512i) __builtin_ia32_pbroadcastq512_mem_mask (__A,
80 _mm512_setzero_si512 (),
85 static __inline __m512 __DEFAULT_FN_ATTRS
86 _mm512_setzero_ps(void)
88 return (__m512){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
89 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
91 static __inline __m512d __DEFAULT_FN_ATTRS
92 _mm512_setzero_pd(void)
94 return (__m512d){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
97 static __inline __m512 __DEFAULT_FN_ATTRS
98 _mm512_set1_ps(float __w)
100 return (__m512){ __w, __w, __w, __w, __w, __w, __w, __w,
101 __w, __w, __w, __w, __w, __w, __w, __w };
104 static __inline __m512d __DEFAULT_FN_ATTRS
105 _mm512_set1_pd(double __w)
107 return (__m512d){ __w, __w, __w, __w, __w, __w, __w, __w };
110 static __inline __m512i __DEFAULT_FN_ATTRS
111 _mm512_set1_epi32(int __s)
113 return (__m512i)(__v16si){ __s, __s, __s, __s, __s, __s, __s, __s,
114 __s, __s, __s, __s, __s, __s, __s, __s };
117 static __inline __m512i __DEFAULT_FN_ATTRS
118 _mm512_set1_epi64(long long __d)
120 return (__m512i)(__v8di){ __d, __d, __d, __d, __d, __d, __d, __d };
123 static __inline__ __m512 __DEFAULT_FN_ATTRS
124 _mm512_broadcastss_ps(__m128 __X)
127 return (__v16sf){ __f, __f, __f, __f,
130 __f, __f, __f, __f };
133 static __inline__ __m512d __DEFAULT_FN_ATTRS
134 _mm512_broadcastsd_pd(__m128d __X)
137 return (__v8df){ __d, __d, __d, __d,
138 __d, __d, __d, __d };
141 /* Cast between vector types */
143 static __inline __m512d __DEFAULT_FN_ATTRS
144 _mm512_castpd256_pd512(__m256d __a)
146 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, -1, -1, -1, -1);
149 static __inline __m512 __DEFAULT_FN_ATTRS
150 _mm512_castps256_ps512(__m256 __a)
152 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7,
153 -1, -1, -1, -1, -1, -1, -1, -1);
156 static __inline __m128d __DEFAULT_FN_ATTRS
157 _mm512_castpd512_pd128(__m512d __a)
159 return __builtin_shufflevector(__a, __a, 0, 1);
162 static __inline __m128 __DEFAULT_FN_ATTRS
163 _mm512_castps512_ps128(__m512 __a)
165 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3);
168 /* Bitwise operators */
169 static __inline__ __m512i __DEFAULT_FN_ATTRS
170 _mm512_and_epi32(__m512i __a, __m512i __b)
175 static __inline__ __m512i __DEFAULT_FN_ATTRS
176 _mm512_mask_and_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
178 return (__m512i) __builtin_ia32_pandd512_mask((__v16si) __a,
183 static __inline__ __m512i __DEFAULT_FN_ATTRS
184 _mm512_maskz_and_epi32(__mmask16 __k, __m512i __a, __m512i __b)
186 return (__m512i) __builtin_ia32_pandd512_mask((__v16si) __a,
189 _mm512_setzero_si512 (),
193 static __inline__ __m512i __DEFAULT_FN_ATTRS
194 _mm512_and_epi64(__m512i __a, __m512i __b)
199 static __inline__ __m512i __DEFAULT_FN_ATTRS
200 _mm512_mask_and_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
202 return (__m512i) __builtin_ia32_pandq512_mask ((__v8di) __a,
207 static __inline__ __m512i __DEFAULT_FN_ATTRS
208 _mm512_maskz_and_epi64(__mmask8 __k, __m512i __a, __m512i __b)
210 return (__m512i) __builtin_ia32_pandq512_mask ((__v8di) __a,
213 _mm512_setzero_si512 (),
217 static __inline__ __m512i __DEFAULT_FN_ATTRS
218 _mm512_andnot_epi32 (__m512i __A, __m512i __B)
220 return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A,
223 _mm512_setzero_si512 (),
227 static __inline__ __m512i __DEFAULT_FN_ATTRS
228 _mm512_mask_andnot_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
230 return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A,
236 static __inline__ __m512i __DEFAULT_FN_ATTRS
237 _mm512_maskz_andnot_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
239 return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A,
242 _mm512_setzero_si512 (),
246 static __inline__ __m512i __DEFAULT_FN_ATTRS
247 _mm512_andnot_epi64 (__m512i __A, __m512i __B)
249 return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A,
252 _mm512_setzero_si512 (),
256 static __inline__ __m512i __DEFAULT_FN_ATTRS
257 _mm512_mask_andnot_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
259 return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A,
264 static __inline__ __m512i __DEFAULT_FN_ATTRS
265 _mm512_maskz_andnot_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
267 return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A,
270 _mm512_setzero_pd (),
273 static __inline__ __m512i __DEFAULT_FN_ATTRS
274 _mm512_or_epi32(__m512i __a, __m512i __b)
279 static __inline__ __m512i __DEFAULT_FN_ATTRS
280 _mm512_mask_or_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
282 return (__m512i) __builtin_ia32_pord512_mask((__v16si) __a,
287 static __inline__ __m512i __DEFAULT_FN_ATTRS
288 _mm512_maskz_or_epi32(__mmask16 __k, __m512i __a, __m512i __b)
290 return (__m512i) __builtin_ia32_pord512_mask((__v16si) __a,
293 _mm512_setzero_si512 (),
297 static __inline__ __m512i __DEFAULT_FN_ATTRS
298 _mm512_or_epi64(__m512i __a, __m512i __b)
303 static __inline__ __m512i __DEFAULT_FN_ATTRS
304 _mm512_mask_or_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
306 return (__m512i) __builtin_ia32_porq512_mask ((__v8di) __a,
311 static __inline__ __m512i __DEFAULT_FN_ATTRS
312 _mm512_maskz_or_epi64(__mmask8 __k, __m512i __a, __m512i __b)
314 return (__m512i) __builtin_ia32_porq512_mask ((__v8di) __a,
317 _mm512_setzero_si512 (),
321 static __inline__ __m512i __DEFAULT_FN_ATTRS
322 _mm512_xor_epi32(__m512i __a, __m512i __b)
327 static __inline__ __m512i __DEFAULT_FN_ATTRS
328 _mm512_mask_xor_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
330 return (__m512i) __builtin_ia32_pxord512_mask((__v16si) __a,
335 static __inline__ __m512i __DEFAULT_FN_ATTRS
336 _mm512_maskz_xor_epi32(__mmask16 __k, __m512i __a, __m512i __b)
338 return (__m512i) __builtin_ia32_pxord512_mask((__v16si) __a,
341 _mm512_setzero_si512 (),
345 static __inline__ __m512i __DEFAULT_FN_ATTRS
346 _mm512_xor_epi64(__m512i __a, __m512i __b)
351 static __inline__ __m512i __DEFAULT_FN_ATTRS
352 _mm512_mask_xor_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
354 return (__m512i) __builtin_ia32_pxorq512_mask ((__v8di) __a,
359 static __inline__ __m512i __DEFAULT_FN_ATTRS
360 _mm512_maskz_xor_epi64(__mmask8 __k, __m512i __a, __m512i __b)
362 return (__m512i) __builtin_ia32_pxorq512_mask ((__v8di) __a,
365 _mm512_setzero_si512 (),
369 static __inline__ __m512i __DEFAULT_FN_ATTRS
370 _mm512_and_si512(__m512i __a, __m512i __b)
375 static __inline__ __m512i __DEFAULT_FN_ATTRS
376 _mm512_or_si512(__m512i __a, __m512i __b)
381 static __inline__ __m512i __DEFAULT_FN_ATTRS
382 _mm512_xor_si512(__m512i __a, __m512i __b)
388 static __inline __m512d __DEFAULT_FN_ATTRS
389 _mm512_add_pd(__m512d __a, __m512d __b)
394 static __inline __m512 __DEFAULT_FN_ATTRS
395 _mm512_add_ps(__m512 __a, __m512 __b)
400 static __inline __m512d __DEFAULT_FN_ATTRS
401 _mm512_mul_pd(__m512d __a, __m512d __b)
406 static __inline __m512 __DEFAULT_FN_ATTRS
407 _mm512_mul_ps(__m512 __a, __m512 __b)
412 static __inline __m512d __DEFAULT_FN_ATTRS
413 _mm512_sub_pd(__m512d __a, __m512d __b)
418 static __inline __m512 __DEFAULT_FN_ATTRS
419 _mm512_sub_ps(__m512 __a, __m512 __b)
424 static __inline__ __m512i __DEFAULT_FN_ATTRS
425 _mm512_add_epi64 (__m512i __A, __m512i __B)
427 return (__m512i) ((__v8di) __A + (__v8di) __B);
430 static __inline__ __m512i __DEFAULT_FN_ATTRS
431 _mm512_mask_add_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
433 return (__m512i) __builtin_ia32_paddq512_mask ((__v8di) __A,
439 static __inline__ __m512i __DEFAULT_FN_ATTRS
440 _mm512_maskz_add_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
442 return (__m512i) __builtin_ia32_paddq512_mask ((__v8di) __A,
445 _mm512_setzero_si512 (),
449 static __inline__ __m512i __DEFAULT_FN_ATTRS
450 _mm512_sub_epi64 (__m512i __A, __m512i __B)
452 return (__m512i) ((__v8di) __A - (__v8di) __B);
455 static __inline__ __m512i __DEFAULT_FN_ATTRS
456 _mm512_mask_sub_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
458 return (__m512i) __builtin_ia32_psubq512_mask ((__v8di) __A,
464 static __inline__ __m512i __DEFAULT_FN_ATTRS
465 _mm512_maskz_sub_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
467 return (__m512i) __builtin_ia32_psubq512_mask ((__v8di) __A,
470 _mm512_setzero_si512 (),
474 static __inline__ __m512i __DEFAULT_FN_ATTRS
475 _mm512_add_epi32 (__m512i __A, __m512i __B)
477 return (__m512i) ((__v16si) __A + (__v16si) __B);
480 static __inline__ __m512i __DEFAULT_FN_ATTRS
481 _mm512_mask_add_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
483 return (__m512i) __builtin_ia32_paddd512_mask ((__v16si) __A,
489 static __inline__ __m512i __DEFAULT_FN_ATTRS
490 _mm512_maskz_add_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
492 return (__m512i) __builtin_ia32_paddd512_mask ((__v16si) __A,
495 _mm512_setzero_si512 (),
499 static __inline__ __m512i __DEFAULT_FN_ATTRS
500 _mm512_sub_epi32 (__m512i __A, __m512i __B)
502 return (__m512i) ((__v16si) __A - (__v16si) __B);
505 static __inline__ __m512i __DEFAULT_FN_ATTRS
506 _mm512_mask_sub_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
508 return (__m512i) __builtin_ia32_psubd512_mask ((__v16si) __A,
514 static __inline__ __m512i __DEFAULT_FN_ATTRS
515 _mm512_maskz_sub_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
517 return (__m512i) __builtin_ia32_psubd512_mask ((__v16si) __A,
520 _mm512_setzero_si512 (),
524 static __inline__ __m512d __DEFAULT_FN_ATTRS
525 _mm512_max_pd(__m512d __A, __m512d __B)
527 return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A,
530 _mm512_setzero_pd (),
532 _MM_FROUND_CUR_DIRECTION);
535 static __inline__ __m512 __DEFAULT_FN_ATTRS
536 _mm512_max_ps(__m512 __A, __m512 __B)
538 return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A,
541 _mm512_setzero_ps (),
543 _MM_FROUND_CUR_DIRECTION);
546 static __inline __m512i
548 _mm512_max_epi32(__m512i __A, __m512i __B)
550 return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A,
553 _mm512_setzero_si512 (),
557 static __inline __m512i __DEFAULT_FN_ATTRS
558 _mm512_max_epu32(__m512i __A, __m512i __B)
560 return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A,
563 _mm512_setzero_si512 (),
567 static __inline __m512i __DEFAULT_FN_ATTRS
568 _mm512_max_epi64(__m512i __A, __m512i __B)
570 return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A,
573 _mm512_setzero_si512 (),
577 static __inline __m512i __DEFAULT_FN_ATTRS
578 _mm512_max_epu64(__m512i __A, __m512i __B)
580 return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A,
583 _mm512_setzero_si512 (),
587 static __inline__ __m512d __DEFAULT_FN_ATTRS
588 _mm512_min_pd(__m512d __A, __m512d __B)
590 return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A,
593 _mm512_setzero_pd (),
595 _MM_FROUND_CUR_DIRECTION);
598 static __inline__ __m512 __DEFAULT_FN_ATTRS
599 _mm512_min_ps(__m512 __A, __m512 __B)
601 return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A,
604 _mm512_setzero_ps (),
606 _MM_FROUND_CUR_DIRECTION);
609 static __inline __m512i
611 _mm512_min_epi32(__m512i __A, __m512i __B)
613 return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A,
616 _mm512_setzero_si512 (),
620 static __inline __m512i __DEFAULT_FN_ATTRS
621 _mm512_min_epu32(__m512i __A, __m512i __B)
623 return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A,
626 _mm512_setzero_si512 (),
630 static __inline __m512i __DEFAULT_FN_ATTRS
631 _mm512_min_epi64(__m512i __A, __m512i __B)
633 return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A,
636 _mm512_setzero_si512 (),
640 static __inline __m512i __DEFAULT_FN_ATTRS
641 _mm512_min_epu64(__m512i __A, __m512i __B)
643 return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A,
646 _mm512_setzero_si512 (),
650 static __inline __m512i __DEFAULT_FN_ATTRS
651 _mm512_mul_epi32(__m512i __X, __m512i __Y)
653 return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X,
656 _mm512_setzero_si512 (),
660 static __inline __m512i __DEFAULT_FN_ATTRS
661 _mm512_mask_mul_epi32 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
663 return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X,
668 static __inline __m512i __DEFAULT_FN_ATTRS
669 _mm512_maskz_mul_epi32 (__mmask8 __M, __m512i __X, __m512i __Y)
671 return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X,
674 _mm512_setzero_si512 (),
678 static __inline __m512i __DEFAULT_FN_ATTRS
679 _mm512_mul_epu32(__m512i __X, __m512i __Y)
681 return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X,
684 _mm512_setzero_si512 (),
688 static __inline __m512i __DEFAULT_FN_ATTRS
689 _mm512_mask_mul_epu32 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
691 return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X,
696 static __inline __m512i __DEFAULT_FN_ATTRS
697 _mm512_maskz_mul_epu32 (__mmask8 __M, __m512i __X, __m512i __Y)
699 return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X,
702 _mm512_setzero_si512 (),
706 static __inline __m512i __DEFAULT_FN_ATTRS
707 _mm512_mullo_epi32 (__m512i __A, __m512i __B)
709 return (__m512i) ((__v16si) __A * (__v16si) __B);
712 static __inline __m512i __DEFAULT_FN_ATTRS
713 _mm512_maskz_mullo_epi32 (__mmask16 __M, __m512i __A, __m512i __B)
715 return (__m512i) __builtin_ia32_pmulld512_mask ((__v16si) __A,
718 _mm512_setzero_si512 (),
722 static __inline __m512i __DEFAULT_FN_ATTRS
723 _mm512_mask_mullo_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
725 return (__m512i) __builtin_ia32_pmulld512_mask ((__v16si) __A,
730 static __inline__ __m512d __DEFAULT_FN_ATTRS
731 _mm512_sqrt_pd(__m512d a)
733 return (__m512d)__builtin_ia32_sqrtpd512_mask((__v8df)a,
734 (__v8df) _mm512_setzero_pd (),
736 _MM_FROUND_CUR_DIRECTION);
739 static __inline__ __m512 __DEFAULT_FN_ATTRS
740 _mm512_sqrt_ps(__m512 a)
742 return (__m512)__builtin_ia32_sqrtps512_mask((__v16sf)a,
743 (__v16sf) _mm512_setzero_ps (),
745 _MM_FROUND_CUR_DIRECTION);
748 static __inline__ __m512d __DEFAULT_FN_ATTRS
749 _mm512_rsqrt14_pd(__m512d __A)
751 return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A,
753 _mm512_setzero_pd (),
756 static __inline__ __m512 __DEFAULT_FN_ATTRS
757 _mm512_rsqrt14_ps(__m512 __A)
759 return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A,
761 _mm512_setzero_ps (),
765 static __inline__ __m128 __DEFAULT_FN_ATTRS
766 _mm_rsqrt14_ss(__m128 __A, __m128 __B)
768 return (__m128) __builtin_ia32_rsqrt14ss_mask ((__v4sf) __A,
775 static __inline__ __m128d __DEFAULT_FN_ATTRS
776 _mm_rsqrt14_sd(__m128d __A, __m128d __B)
778 return (__m128d) __builtin_ia32_rsqrt14sd_mask ((__v2df) __A,
785 static __inline__ __m512d __DEFAULT_FN_ATTRS
786 _mm512_rcp14_pd(__m512d __A)
788 return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A,
790 _mm512_setzero_pd (),
794 static __inline__ __m512 __DEFAULT_FN_ATTRS
795 _mm512_rcp14_ps(__m512 __A)
797 return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A,
799 _mm512_setzero_ps (),
802 static __inline__ __m128 __DEFAULT_FN_ATTRS
803 _mm_rcp14_ss(__m128 __A, __m128 __B)
805 return (__m128) __builtin_ia32_rcp14ss_mask ((__v4sf) __A,
812 static __inline__ __m128d __DEFAULT_FN_ATTRS
813 _mm_rcp14_sd(__m128d __A, __m128d __B)
815 return (__m128d) __builtin_ia32_rcp14sd_mask ((__v2df) __A,
822 static __inline __m512 __DEFAULT_FN_ATTRS
823 _mm512_floor_ps(__m512 __A)
825 return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A,
828 _MM_FROUND_CUR_DIRECTION);
831 static __inline __m512d __DEFAULT_FN_ATTRS
832 _mm512_floor_pd(__m512d __A)
834 return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A,
837 _MM_FROUND_CUR_DIRECTION);
840 static __inline __m512 __DEFAULT_FN_ATTRS
841 _mm512_ceil_ps(__m512 __A)
843 return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A,
846 _MM_FROUND_CUR_DIRECTION);
849 static __inline __m512d __DEFAULT_FN_ATTRS
850 _mm512_ceil_pd(__m512d __A)
852 return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A,
855 _MM_FROUND_CUR_DIRECTION);
858 static __inline __m512i __DEFAULT_FN_ATTRS
859 _mm512_abs_epi64(__m512i __A)
861 return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A,
863 _mm512_setzero_si512 (),
867 static __inline __m512i __DEFAULT_FN_ATTRS
868 _mm512_abs_epi32(__m512i __A)
870 return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A,
872 _mm512_setzero_si512 (),
876 #define _mm512_roundscale_ps(A, B) __extension__ ({ \
877 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(A), (B), (__v16sf)(A), \
878 -1, _MM_FROUND_CUR_DIRECTION); })
880 #define _mm512_roundscale_pd(A, B) __extension__ ({ \
881 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(A), (B), (__v8df)(A), \
882 -1, _MM_FROUND_CUR_DIRECTION); })
884 #define _mm512_fmadd_round_pd(A, B, C, R) __extension__ ({ \
885 (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) (A), \
886 (__v8df) (B), (__v8df) (C), \
887 (__mmask8) -1, (R)); })
890 #define _mm512_mask_fmadd_round_pd(A, U, B, C, R) __extension__ ({ \
891 (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) (A), \
892 (__v8df) (B), (__v8df) (C), \
893 (__mmask8) (U), (R)); })
896 #define _mm512_mask3_fmadd_round_pd(A, B, C, U, R) __extension__ ({ \
897 (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) (A), \
898 (__v8df) (B), (__v8df) (C), \
899 (__mmask8) (U), (R)); })
902 #define _mm512_maskz_fmadd_round_pd(U, A, B, C, R) __extension__ ({ \
903 (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) (A), \
904 (__v8df) (B), (__v8df) (C), \
905 (__mmask8) (U), (R)); })
908 #define _mm512_fmsub_round_pd(A, B, C, R) __extension__ ({ \
909 (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) (A), \
910 (__v8df) (B), -(__v8df) (C), \
911 (__mmask8) -1, (R)); })
914 #define _mm512_mask_fmsub_round_pd(A, U, B, C, R) __extension__ ({ \
915 (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) (A), \
916 (__v8df) (B), -(__v8df) (C), \
917 (__mmask8) (U), (R)); })
920 #define _mm512_maskz_fmsub_round_pd(U, A, B, C, R) __extension__ ({ \
921 (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) (A), \
922 (__v8df) (B), -(__v8df) (C), \
923 (__mmask8) (U), (R)); })
926 #define _mm512_fnmadd_round_pd(A, B, C, R) __extension__ ({ \
927 (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) (A), \
928 (__v8df) (B), (__v8df) (C), \
929 (__mmask8) -1, (R)); })
932 #define _mm512_mask3_fnmadd_round_pd(A, B, C, U, R) __extension__ ({ \
933 (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) (A), \
934 (__v8df) (B), (__v8df) (C), \
935 (__mmask8) (U), (R)); })
938 #define _mm512_maskz_fnmadd_round_pd(U, A, B, C, R) __extension__ ({ \
939 (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) (A), \
940 (__v8df) (B), (__v8df) (C), \
941 (__mmask8) (U), (R)); })
944 #define _mm512_fnmsub_round_pd(A, B, C, R) __extension__ ({ \
945 (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) (A), \
946 (__v8df) (B), -(__v8df) (C), \
947 (__mmask8) -1, (R)); })
950 #define _mm512_maskz_fnmsub_round_pd(U, A, B, C, R) __extension__ ({ \
951 (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) (A), \
952 (__v8df) (B), -(__v8df) (C), \
953 (__mmask8) (U), (R)); })
956 static __inline__ __m512d __DEFAULT_FN_ATTRS
957 _mm512_fmadd_pd(__m512d __A, __m512d __B, __m512d __C)
959 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
963 _MM_FROUND_CUR_DIRECTION);
966 static __inline__ __m512d __DEFAULT_FN_ATTRS
967 _mm512_mask_fmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
969 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
973 _MM_FROUND_CUR_DIRECTION);
976 static __inline__ __m512d __DEFAULT_FN_ATTRS
977 _mm512_mask3_fmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
979 return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A,
983 _MM_FROUND_CUR_DIRECTION);
986 static __inline__ __m512d __DEFAULT_FN_ATTRS
987 _mm512_maskz_fmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
989 return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A,
993 _MM_FROUND_CUR_DIRECTION);
996 static __inline__ __m512d __DEFAULT_FN_ATTRS
997 _mm512_fmsub_pd(__m512d __A, __m512d __B, __m512d __C)
999 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
1003 _MM_FROUND_CUR_DIRECTION);
1006 static __inline__ __m512d __DEFAULT_FN_ATTRS
1007 _mm512_mask_fmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
1009 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
1013 _MM_FROUND_CUR_DIRECTION);
1016 static __inline__ __m512d __DEFAULT_FN_ATTRS
1017 _mm512_maskz_fmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
1019 return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A,
1023 _MM_FROUND_CUR_DIRECTION);
1026 static __inline__ __m512d __DEFAULT_FN_ATTRS
1027 _mm512_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C)
1029 return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A,
1033 _MM_FROUND_CUR_DIRECTION);
1036 static __inline__ __m512d __DEFAULT_FN_ATTRS
1037 _mm512_mask3_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
1039 return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A,
1043 _MM_FROUND_CUR_DIRECTION);
1046 static __inline__ __m512d __DEFAULT_FN_ATTRS
1047 _mm512_maskz_fnmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
1049 return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A,
1053 _MM_FROUND_CUR_DIRECTION);
1056 static __inline__ __m512d __DEFAULT_FN_ATTRS
1057 _mm512_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C)
1059 return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A,
1063 _MM_FROUND_CUR_DIRECTION);
1066 static __inline__ __m512d __DEFAULT_FN_ATTRS
1067 _mm512_maskz_fnmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
1069 return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A,
1073 _MM_FROUND_CUR_DIRECTION);
1076 #define _mm512_fmadd_round_ps(A, B, C, R) __extension__ ({ \
1077 (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) (A), \
1078 (__v16sf) (B), (__v16sf) (C), \
1079 (__mmask16) -1, (R)); })
1082 #define _mm512_mask_fmadd_round_ps(A, U, B, C, R) __extension__ ({ \
1083 (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) (A), \
1084 (__v16sf) (B), (__v16sf) (C), \
1085 (__mmask16) (U), (R)); })
1088 #define _mm512_mask3_fmadd_round_ps(A, B, C, U, R) __extension__ ({ \
1089 (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) (A), \
1090 (__v16sf) (B), (__v16sf) (C), \
1091 (__mmask16) (U), (R)); })
1094 #define _mm512_maskz_fmadd_round_ps(U, A, B, C, R) __extension__ ({ \
1095 (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) (A), \
1096 (__v16sf) (B), (__v16sf) (C), \
1097 (__mmask16) (U), (R)); })
1100 #define _mm512_fmsub_round_ps(A, B, C, R) __extension__ ({ \
1101 (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) (A), \
1102 (__v16sf) (B), -(__v16sf) (C), \
1103 (__mmask16) -1, (R)); })
1106 #define _mm512_mask_fmsub_round_ps(A, U, B, C, R) __extension__ ({ \
1107 (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) (A), \
1108 (__v16sf) (B), -(__v16sf) (C), \
1109 (__mmask16) (U), (R)); })
1112 #define _mm512_maskz_fmsub_round_ps(U, A, B, C, R) __extension__ ({ \
1113 (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) (A), \
1114 (__v16sf) (B), -(__v16sf) (C), \
1115 (__mmask16) (U), (R)); })
1118 #define _mm512_fnmadd_round_ps(A, B, C, R) __extension__ ({ \
1119 (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) (A), \
1120 (__v16sf) (B), (__v16sf) (C), \
1121 (__mmask16) -1, (R)); })
1124 #define _mm512_mask3_fnmadd_round_ps(A, B, C, U, R) __extension__ ({ \
1125 (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) (A), \
1126 (__v16sf) (B), (__v16sf) (C), \
1127 (__mmask16) (U), (R)); })
1130 #define _mm512_maskz_fnmadd_round_ps(U, A, B, C, R) __extension__ ({ \
1131 (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) (A), \
1132 (__v16sf) (B), (__v16sf) (C), \
1133 (__mmask16) (U), (R)); })
1136 #define _mm512_fnmsub_round_ps(A, B, C, R) __extension__ ({ \
1137 (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) (A), \
1138 (__v16sf) (B), -(__v16sf) (C), \
1139 (__mmask16) -1, (R)); })
1142 #define _mm512_maskz_fnmsub_round_ps(U, A, B, C, R) __extension__ ({ \
1143 (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) (A), \
1144 (__v16sf) (B), -(__v16sf) (C), \
1145 (__mmask16) (U), (R)); })
1148 static __inline__ __m512 __DEFAULT_FN_ATTRS
1149 _mm512_fmadd_ps(__m512 __A, __m512 __B, __m512 __C)
1151 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
1155 _MM_FROUND_CUR_DIRECTION);
1158 static __inline__ __m512 __DEFAULT_FN_ATTRS
1159 _mm512_mask_fmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
1161 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
1165 _MM_FROUND_CUR_DIRECTION);
1168 static __inline__ __m512 __DEFAULT_FN_ATTRS
1169 _mm512_mask3_fmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
1171 return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A,
1175 _MM_FROUND_CUR_DIRECTION);
1178 static __inline__ __m512 __DEFAULT_FN_ATTRS
1179 _mm512_maskz_fmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
1181 return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A,
1185 _MM_FROUND_CUR_DIRECTION);
1188 static __inline__ __m512 __DEFAULT_FN_ATTRS
1189 _mm512_fmsub_ps(__m512 __A, __m512 __B, __m512 __C)
1191 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
1195 _MM_FROUND_CUR_DIRECTION);
1198 static __inline__ __m512 __DEFAULT_FN_ATTRS
1199 _mm512_mask_fmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
1201 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
1205 _MM_FROUND_CUR_DIRECTION);
1208 static __inline__ __m512 __DEFAULT_FN_ATTRS
1209 _mm512_maskz_fmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
1211 return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A,
1215 _MM_FROUND_CUR_DIRECTION);
1218 static __inline__ __m512 __DEFAULT_FN_ATTRS
1219 _mm512_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C)
1221 return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A,
1225 _MM_FROUND_CUR_DIRECTION);
1228 static __inline__ __m512 __DEFAULT_FN_ATTRS
1229 _mm512_mask3_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
1231 return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A,
1235 _MM_FROUND_CUR_DIRECTION);
1238 static __inline__ __m512 __DEFAULT_FN_ATTRS
1239 _mm512_maskz_fnmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
1241 return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A,
1245 _MM_FROUND_CUR_DIRECTION);
1248 static __inline__ __m512 __DEFAULT_FN_ATTRS
1249 _mm512_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C)
1251 return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A,
1255 _MM_FROUND_CUR_DIRECTION);
1258 static __inline__ __m512 __DEFAULT_FN_ATTRS
1259 _mm512_maskz_fnmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
1261 return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A,
1265 _MM_FROUND_CUR_DIRECTION);
1268 #define _mm512_fmaddsub_round_pd(A, B, C, R) __extension__ ({ \
1269 (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) (A), \
1270 (__v8df) (B), (__v8df) (C), \
1271 (__mmask8) -1, (R)); })
1274 #define _mm512_mask_fmaddsub_round_pd(A, U, B, C, R) __extension__ ({ \
1275 (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) (A), \
1276 (__v8df) (B), (__v8df) (C), \
1277 (__mmask8) (U), (R)); })
1280 #define _mm512_mask3_fmaddsub_round_pd(A, B, C, U, R) __extension__ ({ \
1281 (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) (A), \
1282 (__v8df) (B), (__v8df) (C), \
1283 (__mmask8) (U), (R)); })
1286 #define _mm512_maskz_fmaddsub_round_pd(U, A, B, C, R) __extension__ ({ \
1287 (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) (A), \
1288 (__v8df) (B), (__v8df) (C), \
1289 (__mmask8) (U), (R)); })
1292 #define _mm512_fmsubadd_round_pd(A, B, C, R) __extension__ ({ \
1293 (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) (A), \
1294 (__v8df) (B), -(__v8df) (C), \
1295 (__mmask8) -1, (R)); })
1298 #define _mm512_mask_fmsubadd_round_pd(A, U, B, C, R) __extension__ ({ \
1299 (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) (A), \
1300 (__v8df) (B), -(__v8df) (C), \
1301 (__mmask8) (U), (R)); })
1304 #define _mm512_maskz_fmsubadd_round_pd(U, A, B, C, R) __extension__ ({ \
1305 (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) (A), \
1306 (__v8df) (B), -(__v8df) (C), \
1307 (__mmask8) (U), (R)); })
1310 static __inline__ __m512d __DEFAULT_FN_ATTRS
1311 _mm512_fmaddsub_pd(__m512d __A, __m512d __B, __m512d __C)
1313 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
1317 _MM_FROUND_CUR_DIRECTION);
1320 static __inline__ __m512d __DEFAULT_FN_ATTRS
1321 _mm512_mask_fmaddsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
1323 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
1327 _MM_FROUND_CUR_DIRECTION);
1330 static __inline__ __m512d __DEFAULT_FN_ATTRS
1331 _mm512_mask3_fmaddsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
1333 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) __A,
1337 _MM_FROUND_CUR_DIRECTION);
1340 static __inline__ __m512d __DEFAULT_FN_ATTRS
1341 _mm512_maskz_fmaddsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
1343 return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A,
1347 _MM_FROUND_CUR_DIRECTION);
1350 static __inline__ __m512d __DEFAULT_FN_ATTRS
1351 _mm512_fmsubadd_pd(__m512d __A, __m512d __B, __m512d __C)
1353 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
1357 _MM_FROUND_CUR_DIRECTION);
1360 static __inline__ __m512d __DEFAULT_FN_ATTRS
1361 _mm512_mask_fmsubadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
1363 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
1367 _MM_FROUND_CUR_DIRECTION);
1370 static __inline__ __m512d __DEFAULT_FN_ATTRS
1371 _mm512_maskz_fmsubadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
1373 return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A,
1377 _MM_FROUND_CUR_DIRECTION);
1380 #define _mm512_fmaddsub_round_ps(A, B, C, R) __extension__ ({ \
1381 (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) (A), \
1382 (__v16sf) (B), (__v16sf) (C), \
1383 (__mmask16) -1, (R)); })
1386 #define _mm512_mask_fmaddsub_round_ps(A, U, B, C, R) __extension__ ({ \
1387 (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) (A), \
1388 (__v16sf) (B), (__v16sf) (C), \
1389 (__mmask16) (U), (R)); })
1392 #define _mm512_mask3_fmaddsub_round_ps(A, B, C, U, R) __extension__ ({ \
1393 (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) (A), \
1394 (__v16sf) (B), (__v16sf) (C), \
1395 (__mmask16) (U), (R)); })
1398 #define _mm512_maskz_fmaddsub_round_ps(U, A, B, C, R) __extension__ ({ \
1399 (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) (A), \
1400 (__v16sf) (B), (__v16sf) (C), \
1401 (__mmask16) (U), (R)); })
1404 #define _mm512_fmsubadd_round_ps(A, B, C, R) __extension__ ({ \
1405 (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) (A), \
1406 (__v16sf) (B), -(__v16sf) (C), \
1407 (__mmask16) -1, (R)); })
1410 #define _mm512_mask_fmsubadd_round_ps(A, U, B, C, R) __extension__ ({ \
1411 (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) (A), \
1412 (__v16sf) (B), -(__v16sf) (C), \
1413 (__mmask16) (U), (R)); })
1416 #define _mm512_maskz_fmsubadd_round_ps(U, A, B, C, R) __extension__ ({ \
1417 (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) (A), \
1418 (__v16sf) (B), -(__v16sf) (C), \
1419 (__mmask16) (U), (R)); })
1422 static __inline__ __m512 __DEFAULT_FN_ATTRS
1423 _mm512_fmaddsub_ps(__m512 __A, __m512 __B, __m512 __C)
1425 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
1429 _MM_FROUND_CUR_DIRECTION);
1432 static __inline__ __m512 __DEFAULT_FN_ATTRS
1433 _mm512_mask_fmaddsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
1435 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
1439 _MM_FROUND_CUR_DIRECTION);
1442 static __inline__ __m512 __DEFAULT_FN_ATTRS
1443 _mm512_mask3_fmaddsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
1445 return (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) __A,
1449 _MM_FROUND_CUR_DIRECTION);
1452 static __inline__ __m512 __DEFAULT_FN_ATTRS
1453 _mm512_maskz_fmaddsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
1455 return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A,
1459 _MM_FROUND_CUR_DIRECTION);
1462 static __inline__ __m512 __DEFAULT_FN_ATTRS
1463 _mm512_fmsubadd_ps(__m512 __A, __m512 __B, __m512 __C)
1465 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
1469 _MM_FROUND_CUR_DIRECTION);
1472 static __inline__ __m512 __DEFAULT_FN_ATTRS
1473 _mm512_mask_fmsubadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
1475 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
1479 _MM_FROUND_CUR_DIRECTION);
1482 static __inline__ __m512 __DEFAULT_FN_ATTRS
1483 _mm512_maskz_fmsubadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
1485 return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A,
1489 _MM_FROUND_CUR_DIRECTION);
1492 #define _mm512_mask3_fmsub_round_pd(A, B, C, U, R) __extension__ ({ \
1493 (__m512d) __builtin_ia32_vfmsubpd512_mask3 ((__v8df) (A), \
1494 (__v8df) (B), (__v8df) (C), \
1495 (__mmask8) (U), (R)); })
1498 static __inline__ __m512d __DEFAULT_FN_ATTRS
1499 _mm512_mask3_fmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
1501 return (__m512d) __builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A,
1505 _MM_FROUND_CUR_DIRECTION);
1508 #define _mm512_mask3_fmsub_round_ps(A, B, C, U, R) __extension__ ({ \
1509 (__m512) __builtin_ia32_vfmsubps512_mask3 ((__v16sf) (A), \
1510 (__v16sf) (B), (__v16sf) (C), \
1511 (__mmask16) (U), (R)); })
1514 static __inline__ __m512 __DEFAULT_FN_ATTRS
1515 _mm512_mask3_fmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
1517 return (__m512) __builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A,
1521 _MM_FROUND_CUR_DIRECTION);
1524 #define _mm512_mask3_fmsubadd_round_pd(A, B, C, U, R) __extension__ ({ \
1525 (__m512d) __builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) (A), \
1526 (__v8df) (B), (__v8df) (C), \
1527 (__mmask8) (U), (R)); })
1530 static __inline__ __m512d __DEFAULT_FN_ATTRS
1531 _mm512_mask3_fmsubadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
1533 return (__m512d) __builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) __A,
1537 _MM_FROUND_CUR_DIRECTION);
1540 #define _mm512_mask3_fmsubadd_round_ps(A, B, C, U, R) __extension__ ({ \
1541 (__m512) __builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) (A), \
1542 (__v16sf) (B), (__v16sf) (C), \
1543 (__mmask16) (U), (R)); })
1546 static __inline__ __m512 __DEFAULT_FN_ATTRS
1547 _mm512_mask3_fmsubadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
1549 return (__m512) __builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) __A,
1553 _MM_FROUND_CUR_DIRECTION);
1556 #define _mm512_mask_fnmadd_round_pd(A, U, B, C, R) __extension__ ({ \
1557 (__m512d) __builtin_ia32_vfnmaddpd512_mask ((__v8df) (A), \
1558 (__v8df) (B), (__v8df) (C), \
1559 (__mmask8) (U), (R)); })
1562 static __inline__ __m512d __DEFAULT_FN_ATTRS
1563 _mm512_mask_fnmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
1565 return (__m512d) __builtin_ia32_vfnmaddpd512_mask ((__v8df) __A,
1569 _MM_FROUND_CUR_DIRECTION);
1572 #define _mm512_mask_fnmadd_round_ps(A, U, B, C, R) __extension__ ({ \
1573 (__m512) __builtin_ia32_vfnmaddps512_mask ((__v16sf) (A), \
1574 (__v16sf) (B), (__v16sf) (C), \
1575 (__mmask16) (U), (R)); })
1578 static __inline__ __m512 __DEFAULT_FN_ATTRS
1579 _mm512_mask_fnmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
1581 return (__m512) __builtin_ia32_vfnmaddps512_mask ((__v16sf) __A,
1585 _MM_FROUND_CUR_DIRECTION);
1588 #define _mm512_mask_fnmsub_round_pd(A, U, B, C, R) __extension__ ({ \
1589 (__m512d) __builtin_ia32_vfnmsubpd512_mask ((__v8df) (A), \
1590 (__v8df) (B), (__v8df) (C), \
1591 (__mmask8) (U), (R)); })
1594 #define _mm512_mask3_fnmsub_round_pd(A, B, C, U, R) __extension__ ({ \
1595 (__m512d) __builtin_ia32_vfnmsubpd512_mask3 ((__v8df) (A), \
1596 (__v8df) (B), (__v8df) (C), \
1597 (__mmask8) (U), (R)); })
1600 static __inline__ __m512d __DEFAULT_FN_ATTRS
1601 _mm512_mask_fnmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
1603 return (__m512d) __builtin_ia32_vfnmsubpd512_mask ((__v8df) __A,
1607 _MM_FROUND_CUR_DIRECTION);
1610 static __inline__ __m512d __DEFAULT_FN_ATTRS
1611 _mm512_mask3_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
1613 return (__m512d) __builtin_ia32_vfnmsubpd512_mask3 ((__v8df) __A,
1617 _MM_FROUND_CUR_DIRECTION);
1620 #define _mm512_mask_fnmsub_round_ps(A, U, B, C, R) __extension__ ({ \
1621 (__m512) __builtin_ia32_vfnmsubps512_mask ((__v16sf) (A), \
1622 (__v16sf) (B), (__v16sf) (C), \
1623 (__mmask16) (U), (R)); })
1626 #define _mm512_mask3_fnmsub_round_ps(A, B, C, U, R) __extension__ ({ \
1627 (__m512) __builtin_ia32_vfnmsubps512_mask3 ((__v16sf) (A), \
1628 (__v16sf) (B), (__v16sf) (C), \
1629 (__mmask16) (U), (R)); })
1632 static __inline__ __m512 __DEFAULT_FN_ATTRS
1633 _mm512_mask_fnmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
1635 return (__m512) __builtin_ia32_vfnmsubps512_mask ((__v16sf) __A,
1639 _MM_FROUND_CUR_DIRECTION);
1642 static __inline__ __m512 __DEFAULT_FN_ATTRS
1643 _mm512_mask3_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
1645 return (__m512) __builtin_ia32_vfnmsubps512_mask3 ((__v16sf) __A,
1649 _MM_FROUND_CUR_DIRECTION);
1654 /* Vector permutations */
1656 static __inline __m512i __DEFAULT_FN_ATTRS
1657 _mm512_permutex2var_epi32(__m512i __A, __m512i __I, __m512i __B)
1659 return (__m512i) __builtin_ia32_vpermt2vard512_mask ((__v16si) __I
1665 static __inline __m512i __DEFAULT_FN_ATTRS
1666 _mm512_permutex2var_epi64(__m512i __A, __m512i __I, __m512i __B)
1668 return (__m512i) __builtin_ia32_vpermt2varq512_mask ((__v8di) __I
1675 static __inline __m512d __DEFAULT_FN_ATTRS
1676 _mm512_permutex2var_pd(__m512d __A, __m512i __I, __m512d __B)
1678 return (__m512d) __builtin_ia32_vpermt2varpd512_mask ((__v8di) __I
1684 static __inline __m512 __DEFAULT_FN_ATTRS
1685 _mm512_permutex2var_ps(__m512 __A, __m512i __I, __m512 __B)
1687 return (__m512) __builtin_ia32_vpermt2varps512_mask ((__v16si) __I
1694 #define _mm512_alignr_epi64(A, B, I) __extension__ ({ \
1695 (__m512i)__builtin_ia32_alignq512_mask((__v8di)(__m512i)(A), \
1696 (__v8di)(__m512i)(B), \
1697 (I), (__v8di)_mm512_setzero_si512(), \
1700 #define _mm512_alignr_epi32(A, B, I) __extension__ ({ \
1701 (__m512i)__builtin_ia32_alignd512_mask((__v16si)(__m512i)(A), \
1702 (__v16si)(__m512i)(B), \
1703 (I), (__v16si)_mm512_setzero_si512(), \
1706 /* Vector Extract */
1708 #define _mm512_extractf64x4_pd(A, I) __extension__ ({ \
1709 __m512d __A = (A); \
1711 __builtin_ia32_extractf64x4_mask((__v8df)__A, \
1713 (__v4df)_mm256_setzero_si256(), \
1716 #define _mm512_extractf32x4_ps(A, I) __extension__ ({ \
1719 __builtin_ia32_extractf32x4_mask((__v16sf)__A, \
1721 (__v4sf)_mm_setzero_ps(), \
1726 static __inline __m512d __DEFAULT_FN_ATTRS
1727 _mm512_mask_blend_pd(__mmask8 __U, __m512d __A, __m512d __W)
1729 return (__m512d) __builtin_ia32_blendmpd_512_mask ((__v8df) __A,
1734 static __inline __m512 __DEFAULT_FN_ATTRS
1735 _mm512_mask_blend_ps(__mmask16 __U, __m512 __A, __m512 __W)
1737 return (__m512) __builtin_ia32_blendmps_512_mask ((__v16sf) __A,
1742 static __inline __m512i __DEFAULT_FN_ATTRS
1743 _mm512_mask_blend_epi64(__mmask8 __U, __m512i __A, __m512i __W)
1745 return (__m512i) __builtin_ia32_blendmq_512_mask ((__v8di) __A,
1750 static __inline __m512i __DEFAULT_FN_ATTRS
1751 _mm512_mask_blend_epi32(__mmask16 __U, __m512i __A, __m512i __W)
1753 return (__m512i) __builtin_ia32_blendmd_512_mask ((__v16si) __A,
1760 #define _mm512_cmp_round_ps_mask(A, B, P, R) __extension__ ({ \
1761 (__mmask16)__builtin_ia32_cmpps512_mask((__v16sf)(__m512)(A), \
1762 (__v16sf)(__m512)(B), \
1763 (P), (__mmask16)-1, (R)); })
1765 #define _mm512_mask_cmp_round_ps_mask(U, A, B, P, R) __extension__ ({ \
1766 (__mmask16)__builtin_ia32_cmpps512_mask((__v16sf)(__m512)(A), \
1767 (__v16sf)(__m512)(B), \
1768 (P), (__mmask16)(U), (R)); })
1770 #define _mm512_cmp_ps_mask(A, B, P) \
1771 _mm512_cmp_round_ps_mask((A), (B), (P), _MM_FROUND_CUR_DIRECTION)
1773 #define _mm512_mask_cmp_ps_mask(U, A, B, P) \
1774 _mm512_mask_cmp_round_ps_mask((U), (A), (B), (P), _MM_FROUND_CUR_DIRECTION)
1776 #define _mm512_cmp_round_pd_mask(A, B, P, R) __extension__ ({ \
1777 (__mmask8)__builtin_ia32_cmppd512_mask((__v8df)(__m512d)(A), \
1778 (__v8df)(__m512d)(B), \
1779 (P), (__mmask8)-1, (R)); })
1781 #define _mm512_mask_cmp_round_pd_mask(U, A, B, P, R) __extension__ ({ \
1782 (__mmask8)__builtin_ia32_cmppd512_mask((__v8df)(__m512d)(A), \
1783 (__v8df)(__m512d)(B), \
1784 (P), (__mmask8)(U), (R)); })
1786 #define _mm512_cmp_pd_mask(A, B, P) \
1787 _mm512_cmp_round_pd_mask((A), (B), (P), _MM_FROUND_CUR_DIRECTION)
1789 #define _mm512_mask_cmp_pd_mask(U, A, B, P) \
1790 _mm512_mask_cmp_round_pd_mask((U), (A), (B), (P), _MM_FROUND_CUR_DIRECTION)
1794 static __inline __m512i __DEFAULT_FN_ATTRS
1795 _mm512_cvttps_epu32(__m512 __A)
1797 return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A,
1799 _mm512_setzero_si512 (),
1801 _MM_FROUND_CUR_DIRECTION);
1804 #define _mm512_cvt_roundepi32_ps(A, R) __extension__ ({ \
1805 (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), \
1806 (__v16sf)_mm512_setzero_ps(), \
1807 (__mmask16)-1, (R)); })
1809 #define _mm512_cvt_roundepu32_ps(A, R) __extension__ ({ \
1810 (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), \
1811 (__v16sf)_mm512_setzero_ps(), \
1812 (__mmask16)-1, (R)); })
1814 static __inline __m512d __DEFAULT_FN_ATTRS
1815 _mm512_cvtepi32_pd(__m256i __A)
1817 return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A,
1819 _mm512_setzero_pd (),
1823 static __inline __m512d __DEFAULT_FN_ATTRS
1824 _mm512_cvtepu32_pd(__m256i __A)
1826 return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A,
1828 _mm512_setzero_pd (),
1832 #define _mm512_cvt_roundpd_ps(A, R) __extension__ ({ \
1833 (__m256)__builtin_ia32_cvtpd2ps512_mask((__v8df)(A), \
1834 (__v8sf)_mm256_setzero_ps(), \
1835 (__mmask8)-1, (R)); })
1837 #define _mm512_cvtps_ph(A, I) __extension__ ({ \
1838 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(A), (I), \
1839 (__v16hi)_mm256_setzero_si256(), \
1842 static __inline __m512 __DEFAULT_FN_ATTRS
1843 _mm512_cvtph_ps(__m256i __A)
1845 return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A,
1847 _mm512_setzero_ps (),
1849 _MM_FROUND_CUR_DIRECTION);
1852 static __inline __m512i __DEFAULT_FN_ATTRS
1853 _mm512_cvttps_epi32(__m512 a)
1856 __builtin_ia32_cvttps2dq512_mask((__v16sf) a,
1857 (__v16si) _mm512_setzero_si512 (),
1858 (__mmask16) -1, _MM_FROUND_CUR_DIRECTION);
1861 static __inline __m256i __DEFAULT_FN_ATTRS
1862 _mm512_cvttpd_epi32(__m512d a)
1864 return (__m256i)__builtin_ia32_cvttpd2dq512_mask((__v8df) a,
1865 (__v8si)_mm256_setzero_si256(),
1867 _MM_FROUND_CUR_DIRECTION);
1870 #define _mm512_cvtt_roundpd_epi32(A, R) __extension__ ({ \
1871 (__m256i)__builtin_ia32_cvttpd2dq512_mask((__v8df)(A), \
1872 (__v8si)_mm256_setzero_si256(), \
1873 (__mmask8)-1, (R)); })
1875 #define _mm512_cvtt_roundps_epi32(A, R) __extension__ ({ \
1876 (__m512i)__builtin_ia32_cvttps2dq512_mask((__v16sf)(A), \
1877 (__v16si)_mm512_setzero_si512(), \
1878 (__mmask16)-1, (R)); })
1880 #define _mm512_cvt_roundps_epi32(A, R) __extension__ ({ \
1881 (__m512i)__builtin_ia32_cvtps2dq512_mask((__v16sf)(A), \
1882 (__v16si)_mm512_setzero_si512(), \
1883 (__mmask16)-1, (R)); })
1885 #define _mm512_cvt_roundpd_epi32(A, R) __extension__ ({ \
1886 (__m256i)__builtin_ia32_cvtpd2dq512_mask((__v8df)(A), \
1887 (__v8si)_mm256_setzero_si256(), \
1888 (__mmask8)-1, (R)); })
1890 #define _mm512_cvt_roundps_epu32(A, R) __extension__ ({ \
1891 (__m512i)__builtin_ia32_cvtps2udq512_mask((__v16sf)(A), \
1892 (__v16si)_mm512_setzero_si512(), \
1893 (__mmask16)-1, (R)); })
1895 #define _mm512_cvt_roundpd_epu32(A, R) __extension__ ({ \
1896 (__m256i)__builtin_ia32_cvtpd2udq512_mask((__v8df)(A), \
1897 (__v8si)_mm256_setzero_si256(), \
1898 (__mmask8) -1, (R)); })
1900 /* Unpack and Interleave */
1901 static __inline __m512d __DEFAULT_FN_ATTRS
1902 _mm512_unpackhi_pd(__m512d __a, __m512d __b)
1904 return __builtin_shufflevector(__a, __b, 1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
1907 static __inline __m512d __DEFAULT_FN_ATTRS
1908 _mm512_unpacklo_pd(__m512d __a, __m512d __b)
1910 return __builtin_shufflevector(__a, __b, 0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
1913 static __inline __m512 __DEFAULT_FN_ATTRS
1914 _mm512_unpackhi_ps(__m512 __a, __m512 __b)
1916 return __builtin_shufflevector(__a, __b,
1918 2+4, 18+4, 3+4, 19+4,
1919 2+8, 18+8, 3+8, 19+8,
1920 2+12, 18+12, 3+12, 19+12);
1923 static __inline __m512 __DEFAULT_FN_ATTRS
1924 _mm512_unpacklo_ps(__m512 __a, __m512 __b)
1926 return __builtin_shufflevector(__a, __b,
1928 0+4, 16+4, 1+4, 17+4,
1929 0+8, 16+8, 1+8, 17+8,
1930 0+12, 16+12, 1+12, 17+12);
1935 static __inline __mmask16 __DEFAULT_FN_ATTRS
1936 _mm512_test_epi32_mask(__m512i __A, __m512i __B)
1938 return (__mmask16) __builtin_ia32_ptestmd512 ((__v16si) __A,
1943 static __inline __mmask8 __DEFAULT_FN_ATTRS
1944 _mm512_test_epi64_mask(__m512i __A, __m512i __B)
1946 return (__mmask8) __builtin_ia32_ptestmq512 ((__v8di) __A,
1953 static __inline __m512i __DEFAULT_FN_ATTRS
1954 _mm512_maskz_loadu_epi32(__mmask16 __U, void const *__P)
1956 return (__m512i) __builtin_ia32_loaddqusi512_mask ((const __v16si *)__P,
1958 _mm512_setzero_si512 (),
1962 static __inline __m512i __DEFAULT_FN_ATTRS
1963 _mm512_maskz_loadu_epi64(__mmask8 __U, void const *__P)
1965 return (__m512i) __builtin_ia32_loaddqudi512_mask ((const __v8di *)__P,
1967 _mm512_setzero_si512 (),
1971 static __inline __m512 __DEFAULT_FN_ATTRS
1972 _mm512_maskz_loadu_ps(__mmask16 __U, void const *__P)
1974 return (__m512) __builtin_ia32_loadups512_mask ((const __v16sf *)__P,
1976 _mm512_setzero_ps (),
1980 static __inline __m512d __DEFAULT_FN_ATTRS
1981 _mm512_maskz_loadu_pd(__mmask8 __U, void const *__P)
1983 return (__m512d) __builtin_ia32_loadupd512_mask ((const __v8df *)__P,
1985 _mm512_setzero_pd (),
1989 static __inline __m512 __DEFAULT_FN_ATTRS
1990 _mm512_maskz_load_ps(__mmask16 __U, void const *__P)
1992 return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *)__P,
1994 _mm512_setzero_ps (),
1998 static __inline __m512d __DEFAULT_FN_ATTRS
1999 _mm512_maskz_load_pd(__mmask8 __U, void const *__P)
2001 return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *)__P,
2003 _mm512_setzero_pd (),
2007 static __inline __m512d __DEFAULT_FN_ATTRS
2008 _mm512_loadu_pd(double const *__p)
2012 } __attribute__((__packed__, __may_alias__));
2013 return ((struct __loadu_pd*)__p)->__v;
2016 static __inline __m512 __DEFAULT_FN_ATTRS
2017 _mm512_loadu_ps(float const *__p)
2021 } __attribute__((__packed__, __may_alias__));
2022 return ((struct __loadu_ps*)__p)->__v;
2025 static __inline __m512 __DEFAULT_FN_ATTRS
2026 _mm512_load_ps(double const *__p)
2028 return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *)__p,
2030 _mm512_setzero_ps (),
2034 static __inline __m512d __DEFAULT_FN_ATTRS
2035 _mm512_load_pd(float const *__p)
2037 return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *)__p,
2039 _mm512_setzero_pd (),
2043 /* SIMD store ops */
2045 static __inline void __DEFAULT_FN_ATTRS
2046 _mm512_mask_storeu_epi64(void *__P, __mmask8 __U, __m512i __A)
2048 __builtin_ia32_storedqudi512_mask ((__v8di *)__P, (__v8di) __A,
2052 static __inline void __DEFAULT_FN_ATTRS
2053 _mm512_mask_storeu_epi32(void *__P, __mmask16 __U, __m512i __A)
2055 __builtin_ia32_storedqusi512_mask ((__v16si *)__P, (__v16si) __A,
2059 static __inline void __DEFAULT_FN_ATTRS
2060 _mm512_mask_storeu_pd(void *__P, __mmask8 __U, __m512d __A)
2062 __builtin_ia32_storeupd512_mask ((__v8df *)__P, (__v8df) __A, (__mmask8) __U);
2065 static __inline void __DEFAULT_FN_ATTRS
2066 _mm512_storeu_pd(void *__P, __m512d __A)
2068 __builtin_ia32_storeupd512_mask((__v8df *)__P, (__v8df)__A, (__mmask8)-1);
2071 static __inline void __DEFAULT_FN_ATTRS
2072 _mm512_mask_storeu_ps(void *__P, __mmask16 __U, __m512 __A)
2074 __builtin_ia32_storeups512_mask ((__v16sf *)__P, (__v16sf) __A,
2078 static __inline void __DEFAULT_FN_ATTRS
2079 _mm512_storeu_ps(void *__P, __m512 __A)
2081 __builtin_ia32_storeups512_mask((__v16sf *)__P, (__v16sf)__A, (__mmask16)-1);
2084 static __inline void __DEFAULT_FN_ATTRS
2085 _mm512_mask_store_pd(void *__P, __mmask8 __U, __m512d __A)
2087 __builtin_ia32_storeapd512_mask ((__v8df *)__P, (__v8df) __A, (__mmask8) __U);
2090 static __inline void __DEFAULT_FN_ATTRS
2091 _mm512_store_pd(void *__P, __m512d __A)
2093 *(__m512d*)__P = __A;
2096 static __inline void __DEFAULT_FN_ATTRS
2097 _mm512_mask_store_ps(void *__P, __mmask16 __U, __m512 __A)
2099 __builtin_ia32_storeaps512_mask ((__v16sf *)__P, (__v16sf) __A,
2103 static __inline void __DEFAULT_FN_ATTRS
2104 _mm512_store_ps(void *__P, __m512 __A)
2106 *(__m512*)__P = __A;
2111 static __inline __mmask16 __DEFAULT_FN_ATTRS
2112 _mm512_knot(__mmask16 __M)
2114 return __builtin_ia32_knothi(__M);
2117 /* Integer compare */
2119 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2120 _mm512_cmpeq_epi32_mask(__m512i __a, __m512i __b) {
2121 return (__mmask16)__builtin_ia32_pcmpeqd512_mask((__v16si)__a, (__v16si)__b,
2125 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2126 _mm512_mask_cmpeq_epi32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2127 return (__mmask16)__builtin_ia32_pcmpeqd512_mask((__v16si)__a, (__v16si)__b,
2131 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2132 _mm512_cmpeq_epu32_mask(__m512i __a, __m512i __b) {
2133 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 0,
2137 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2138 _mm512_mask_cmpeq_epu32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2139 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 0,
2143 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2144 _mm512_mask_cmpeq_epi64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2145 return (__mmask8)__builtin_ia32_pcmpeqq512_mask((__v8di)__a, (__v8di)__b,
2149 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2150 _mm512_cmpeq_epi64_mask(__m512i __a, __m512i __b) {
2151 return (__mmask8)__builtin_ia32_pcmpeqq512_mask((__v8di)__a, (__v8di)__b,
2155 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2156 _mm512_cmpeq_epu64_mask(__m512i __a, __m512i __b) {
2157 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 0,
2161 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2162 _mm512_mask_cmpeq_epu64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2163 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 0,
2167 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2168 _mm512_cmpge_epi32_mask(__m512i __a, __m512i __b) {
2169 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 5,
2173 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2174 _mm512_mask_cmpge_epi32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2175 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 5,
2179 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2180 _mm512_cmpge_epu32_mask(__m512i __a, __m512i __b) {
2181 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 5,
2185 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2186 _mm512_mask_cmpge_epu32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2187 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 5,
2191 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2192 _mm512_cmpge_epi64_mask(__m512i __a, __m512i __b) {
2193 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 5,
2197 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2198 _mm512_mask_cmpge_epi64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2199 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 5,
2203 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2204 _mm512_cmpge_epu64_mask(__m512i __a, __m512i __b) {
2205 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 5,
2209 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2210 _mm512_mask_cmpge_epu64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2211 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 5,
2215 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2216 _mm512_cmpgt_epi32_mask(__m512i __a, __m512i __b) {
2217 return (__mmask16)__builtin_ia32_pcmpgtd512_mask((__v16si)__a, (__v16si)__b,
2221 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2222 _mm512_mask_cmpgt_epi32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2223 return (__mmask16)__builtin_ia32_pcmpgtd512_mask((__v16si)__a, (__v16si)__b,
2227 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2228 _mm512_cmpgt_epu32_mask(__m512i __a, __m512i __b) {
2229 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 6,
2233 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2234 _mm512_mask_cmpgt_epu32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2235 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 6,
2239 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2240 _mm512_mask_cmpgt_epi64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2241 return (__mmask8)__builtin_ia32_pcmpgtq512_mask((__v8di)__a, (__v8di)__b,
2245 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2246 _mm512_cmpgt_epi64_mask(__m512i __a, __m512i __b) {
2247 return (__mmask8)__builtin_ia32_pcmpgtq512_mask((__v8di)__a, (__v8di)__b,
2251 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2252 _mm512_cmpgt_epu64_mask(__m512i __a, __m512i __b) {
2253 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 6,
2257 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2258 _mm512_mask_cmpgt_epu64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2259 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 6,
2263 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2264 _mm512_cmple_epi32_mask(__m512i __a, __m512i __b) {
2265 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 2,
2269 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2270 _mm512_mask_cmple_epi32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2271 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 2,
2275 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2276 _mm512_cmple_epu32_mask(__m512i __a, __m512i __b) {
2277 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 2,
2281 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2282 _mm512_mask_cmple_epu32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2283 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 2,
2287 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2288 _mm512_cmple_epi64_mask(__m512i __a, __m512i __b) {
2289 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 2,
2293 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2294 _mm512_mask_cmple_epi64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2295 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 2,
2299 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2300 _mm512_cmple_epu64_mask(__m512i __a, __m512i __b) {
2301 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 2,
2305 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2306 _mm512_mask_cmple_epu64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2307 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 2,
2311 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2312 _mm512_cmplt_epi32_mask(__m512i __a, __m512i __b) {
2313 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 1,
2317 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2318 _mm512_mask_cmplt_epi32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2319 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 1,
2323 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2324 _mm512_cmplt_epu32_mask(__m512i __a, __m512i __b) {
2325 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 1,
2329 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2330 _mm512_mask_cmplt_epu32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2331 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 1,
2335 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2336 _mm512_cmplt_epi64_mask(__m512i __a, __m512i __b) {
2337 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 1,
2341 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2342 _mm512_mask_cmplt_epi64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2343 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 1,
2347 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2348 _mm512_cmplt_epu64_mask(__m512i __a, __m512i __b) {
2349 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 1,
2353 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2354 _mm512_mask_cmplt_epu64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2355 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 1,
2359 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2360 _mm512_cmpneq_epi32_mask(__m512i __a, __m512i __b) {
2361 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 4,
2365 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2366 _mm512_mask_cmpneq_epi32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2367 return (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, 4,
2371 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2372 _mm512_cmpneq_epu32_mask(__m512i __a, __m512i __b) {
2373 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 4,
2377 static __inline__ __mmask16 __DEFAULT_FN_ATTRS
2378 _mm512_mask_cmpneq_epu32_mask(__mmask16 __u, __m512i __a, __m512i __b) {
2379 return (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, 4,
2383 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2384 _mm512_cmpneq_epi64_mask(__m512i __a, __m512i __b) {
2385 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 4,
2389 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2390 _mm512_mask_cmpneq_epi64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2391 return (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, 4,
2395 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2396 _mm512_cmpneq_epu64_mask(__m512i __a, __m512i __b) {
2397 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 4,
2401 static __inline__ __mmask8 __DEFAULT_FN_ATTRS
2402 _mm512_mask_cmpneq_epu64_mask(__mmask8 __u, __m512i __a, __m512i __b) {
2403 return (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, 4,
2407 #define _mm512_cmp_epi32_mask(a, b, p) __extension__ ({ \
2408 __m512i __a = (a); \
2409 __m512i __b = (b); \
2410 (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, (p), \
2413 #define _mm512_cmp_epu32_mask(a, b, p) __extension__ ({ \
2414 __m512i __a = (a); \
2415 __m512i __b = (b); \
2416 (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, (p), \
2419 #define _mm512_cmp_epi64_mask(a, b, p) __extension__ ({ \
2420 __m512i __a = (a); \
2421 __m512i __b = (b); \
2422 (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, (p), \
2425 #define _mm512_cmp_epu64_mask(a, b, p) __extension__ ({ \
2426 __m512i __a = (a); \
2427 __m512i __b = (b); \
2428 (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, (p), \
2431 #define _mm512_mask_cmp_epi32_mask(m, a, b, p) __extension__ ({ \
2432 __m512i __a = (a); \
2433 __m512i __b = (b); \
2434 (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)__a, (__v16si)__b, (p), \
2437 #define _mm512_mask_cmp_epu32_mask(m, a, b, p) __extension__ ({ \
2438 __m512i __a = (a); \
2439 __m512i __b = (b); \
2440 (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)__a, (__v16si)__b, (p), \
2443 #define _mm512_mask_cmp_epi64_mask(m, a, b, p) __extension__ ({ \
2444 __m512i __a = (a); \
2445 __m512i __b = (b); \
2446 (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)__a, (__v8di)__b, (p), \
2449 #define _mm512_mask_cmp_epu64_mask(m, a, b, p) __extension__ ({ \
2450 __m512i __a = (a); \
2451 __m512i __b = (b); \
2452 (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)__a, (__v8di)__b, (p), \
2455 #undef __DEFAULT_FN_ATTRS
2457 #endif // __AVX512FINTRIN_H