1 /*===---- emmintrin.h - SSE2 intrinsics ------------------------------------===
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 *===-----------------------------------------------------------------------===
28 #error "SSE2 instruction set not enabled"
31 #include <xmmintrin.h>
33 typedef double __m128d __attribute__((__vector_size__(16)));
34 typedef long long __m128i __attribute__((__vector_size__(16)));
37 typedef double __v2df __attribute__ ((__vector_size__ (16)));
38 typedef long long __v2di __attribute__ ((__vector_size__ (16)));
39 typedef short __v8hi __attribute__((__vector_size__(16)));
40 typedef char __v16qi __attribute__((__vector_size__(16)));
42 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
43 _mm_add_sd(__m128d __a, __m128d __b)
49 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
50 _mm_add_pd(__m128d __a, __m128d __b)
55 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
56 _mm_sub_sd(__m128d __a, __m128d __b)
62 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
63 _mm_sub_pd(__m128d __a, __m128d __b)
68 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
69 _mm_mul_sd(__m128d __a, __m128d __b)
75 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
76 _mm_mul_pd(__m128d __a, __m128d __b)
81 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
82 _mm_div_sd(__m128d __a, __m128d __b)
88 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
89 _mm_div_pd(__m128d __a, __m128d __b)
94 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
95 _mm_sqrt_sd(__m128d __a, __m128d __b)
97 __m128d __c = __builtin_ia32_sqrtsd(__b);
98 return (__m128d) { __c[0], __a[1] };
101 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
102 _mm_sqrt_pd(__m128d __a)
104 return __builtin_ia32_sqrtpd(__a);
107 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
108 _mm_min_sd(__m128d __a, __m128d __b)
110 return __builtin_ia32_minsd(__a, __b);
113 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
114 _mm_min_pd(__m128d __a, __m128d __b)
116 return __builtin_ia32_minpd(__a, __b);
119 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
120 _mm_max_sd(__m128d __a, __m128d __b)
122 return __builtin_ia32_maxsd(__a, __b);
125 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
126 _mm_max_pd(__m128d __a, __m128d __b)
128 return __builtin_ia32_maxpd(__a, __b);
131 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
132 _mm_and_pd(__m128d __a, __m128d __b)
134 return (__m128d)((__v4si)__a & (__v4si)__b);
137 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
138 _mm_andnot_pd(__m128d __a, __m128d __b)
140 return (__m128d)(~(__v4si)__a & (__v4si)__b);
143 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
144 _mm_or_pd(__m128d __a, __m128d __b)
146 return (__m128d)((__v4si)__a | (__v4si)__b);
149 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
150 _mm_xor_pd(__m128d __a, __m128d __b)
152 return (__m128d)((__v4si)__a ^ (__v4si)__b);
155 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
156 _mm_cmpeq_pd(__m128d __a, __m128d __b)
158 return (__m128d)__builtin_ia32_cmppd(__a, __b, 0);
161 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
162 _mm_cmplt_pd(__m128d __a, __m128d __b)
164 return (__m128d)__builtin_ia32_cmppd(__a, __b, 1);
167 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
168 _mm_cmple_pd(__m128d __a, __m128d __b)
170 return (__m128d)__builtin_ia32_cmppd(__a, __b, 2);
173 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
174 _mm_cmpgt_pd(__m128d __a, __m128d __b)
176 return (__m128d)__builtin_ia32_cmppd(__b, __a, 1);
179 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
180 _mm_cmpge_pd(__m128d __a, __m128d __b)
182 return (__m128d)__builtin_ia32_cmppd(__b, __a, 2);
185 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
186 _mm_cmpord_pd(__m128d __a, __m128d __b)
188 return (__m128d)__builtin_ia32_cmppd(__a, __b, 7);
191 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
192 _mm_cmpunord_pd(__m128d __a, __m128d __b)
194 return (__m128d)__builtin_ia32_cmppd(__a, __b, 3);
197 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
198 _mm_cmpneq_pd(__m128d __a, __m128d __b)
200 return (__m128d)__builtin_ia32_cmppd(__a, __b, 4);
203 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
204 _mm_cmpnlt_pd(__m128d __a, __m128d __b)
206 return (__m128d)__builtin_ia32_cmppd(__a, __b, 5);
209 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
210 _mm_cmpnle_pd(__m128d __a, __m128d __b)
212 return (__m128d)__builtin_ia32_cmppd(__a, __b, 6);
215 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
216 _mm_cmpngt_pd(__m128d __a, __m128d __b)
218 return (__m128d)__builtin_ia32_cmppd(__b, __a, 5);
221 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
222 _mm_cmpnge_pd(__m128d __a, __m128d __b)
224 return (__m128d)__builtin_ia32_cmppd(__b, __a, 6);
227 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
228 _mm_cmpeq_sd(__m128d __a, __m128d __b)
230 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 0);
233 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
234 _mm_cmplt_sd(__m128d __a, __m128d __b)
236 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 1);
239 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
240 _mm_cmple_sd(__m128d __a, __m128d __b)
242 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 2);
245 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
246 _mm_cmpgt_sd(__m128d __a, __m128d __b)
248 return (__m128d)__builtin_ia32_cmpsd(__b, __a, 1);
251 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
252 _mm_cmpge_sd(__m128d __a, __m128d __b)
254 return (__m128d)__builtin_ia32_cmpsd(__b, __a, 2);
257 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
258 _mm_cmpord_sd(__m128d __a, __m128d __b)
260 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 7);
263 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
264 _mm_cmpunord_sd(__m128d __a, __m128d __b)
266 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 3);
269 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
270 _mm_cmpneq_sd(__m128d __a, __m128d __b)
272 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 4);
275 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
276 _mm_cmpnlt_sd(__m128d __a, __m128d __b)
278 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 5);
281 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
282 _mm_cmpnle_sd(__m128d __a, __m128d __b)
284 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 6);
287 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
288 _mm_cmpngt_sd(__m128d __a, __m128d __b)
290 return (__m128d)__builtin_ia32_cmpsd(__b, __a, 5);
293 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
294 _mm_cmpnge_sd(__m128d __a, __m128d __b)
296 return (__m128d)__builtin_ia32_cmpsd(__b, __a, 6);
299 static __inline__ int __attribute__((__always_inline__, __nodebug__))
300 _mm_comieq_sd(__m128d __a, __m128d __b)
302 return __builtin_ia32_comisdeq(__a, __b);
305 static __inline__ int __attribute__((__always_inline__, __nodebug__))
306 _mm_comilt_sd(__m128d __a, __m128d __b)
308 return __builtin_ia32_comisdlt(__a, __b);
311 static __inline__ int __attribute__((__always_inline__, __nodebug__))
312 _mm_comile_sd(__m128d __a, __m128d __b)
314 return __builtin_ia32_comisdle(__a, __b);
317 static __inline__ int __attribute__((__always_inline__, __nodebug__))
318 _mm_comigt_sd(__m128d __a, __m128d __b)
320 return __builtin_ia32_comisdgt(__a, __b);
323 static __inline__ int __attribute__((__always_inline__, __nodebug__))
324 _mm_comige_sd(__m128d __a, __m128d __b)
326 return __builtin_ia32_comisdge(__a, __b);
329 static __inline__ int __attribute__((__always_inline__, __nodebug__))
330 _mm_comineq_sd(__m128d __a, __m128d __b)
332 return __builtin_ia32_comisdneq(__a, __b);
335 static __inline__ int __attribute__((__always_inline__, __nodebug__))
336 _mm_ucomieq_sd(__m128d __a, __m128d __b)
338 return __builtin_ia32_ucomisdeq(__a, __b);
341 static __inline__ int __attribute__((__always_inline__, __nodebug__))
342 _mm_ucomilt_sd(__m128d __a, __m128d __b)
344 return __builtin_ia32_ucomisdlt(__a, __b);
347 static __inline__ int __attribute__((__always_inline__, __nodebug__))
348 _mm_ucomile_sd(__m128d __a, __m128d __b)
350 return __builtin_ia32_ucomisdle(__a, __b);
353 static __inline__ int __attribute__((__always_inline__, __nodebug__))
354 _mm_ucomigt_sd(__m128d __a, __m128d __b)
356 return __builtin_ia32_ucomisdgt(__a, __b);
359 static __inline__ int __attribute__((__always_inline__, __nodebug__))
360 _mm_ucomige_sd(__m128d __a, __m128d __b)
362 return __builtin_ia32_ucomisdge(__a, __b);
365 static __inline__ int __attribute__((__always_inline__, __nodebug__))
366 _mm_ucomineq_sd(__m128d __a, __m128d __b)
368 return __builtin_ia32_ucomisdneq(__a, __b);
371 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
372 _mm_cvtpd_ps(__m128d __a)
374 return __builtin_ia32_cvtpd2ps(__a);
377 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
378 _mm_cvtps_pd(__m128 __a)
380 return __builtin_ia32_cvtps2pd(__a);
383 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
384 _mm_cvtepi32_pd(__m128i __a)
386 return __builtin_ia32_cvtdq2pd((__v4si)__a);
389 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
390 _mm_cvtpd_epi32(__m128d __a)
392 return __builtin_ia32_cvtpd2dq(__a);
395 static __inline__ int __attribute__((__always_inline__, __nodebug__))
396 _mm_cvtsd_si32(__m128d __a)
398 return __builtin_ia32_cvtsd2si(__a);
401 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
402 _mm_cvtsd_ss(__m128 __a, __m128d __b)
408 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
409 _mm_cvtsi32_sd(__m128d __a, int __b)
415 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
416 _mm_cvtss_sd(__m128d __a, __m128 __b)
422 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
423 _mm_cvttpd_epi32(__m128d __a)
425 return (__m128i)__builtin_ia32_cvttpd2dq(__a);
428 static __inline__ int __attribute__((__always_inline__, __nodebug__))
429 _mm_cvttsd_si32(__m128d __a)
434 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
435 _mm_cvtpd_pi32(__m128d __a)
437 return (__m64)__builtin_ia32_cvtpd2pi(__a);
440 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
441 _mm_cvttpd_pi32(__m128d __a)
443 return (__m64)__builtin_ia32_cvttpd2pi(__a);
446 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
447 _mm_cvtpi32_pd(__m64 __a)
449 return __builtin_ia32_cvtpi2pd((__v2si)__a);
452 static __inline__ double __attribute__((__always_inline__, __nodebug__))
453 _mm_cvtsd_f64(__m128d __a)
458 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
459 _mm_load_pd(double const *__dp)
461 return *(__m128d*)__dp;
464 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
465 _mm_load1_pd(double const *__dp)
467 struct __mm_load1_pd_struct {
469 } __attribute__((__packed__, __may_alias__));
470 double __u = ((struct __mm_load1_pd_struct*)__dp)->__u;
471 return (__m128d){ __u, __u };
474 #define _mm_load_pd1(dp) _mm_load1_pd(dp)
476 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
477 _mm_loadr_pd(double const *__dp)
479 __m128d __u = *(__m128d*)__dp;
480 return __builtin_shufflevector(__u, __u, 1, 0);
483 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
484 _mm_loadu_pd(double const *__dp)
488 } __attribute__((packed, may_alias));
489 return ((struct __loadu_pd*)__dp)->__v;
492 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
493 _mm_load_sd(double const *__dp)
495 struct __mm_load_sd_struct {
497 } __attribute__((__packed__, __may_alias__));
498 double __u = ((struct __mm_load_sd_struct*)__dp)->__u;
499 return (__m128d){ __u, 0 };
502 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
503 _mm_loadh_pd(__m128d __a, double const *__dp)
505 struct __mm_loadh_pd_struct {
507 } __attribute__((__packed__, __may_alias__));
508 double __u = ((struct __mm_loadh_pd_struct*)__dp)->__u;
509 return (__m128d){ __a[0], __u };
512 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
513 _mm_loadl_pd(__m128d __a, double const *__dp)
515 struct __mm_loadl_pd_struct {
517 } __attribute__((__packed__, __may_alias__));
518 double __u = ((struct __mm_loadl_pd_struct*)__dp)->__u;
519 return (__m128d){ __u, __a[1] };
522 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
523 _mm_set_sd(double __w)
525 return (__m128d){ __w, 0 };
528 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
529 _mm_set1_pd(double __w)
531 return (__m128d){ __w, __w };
534 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
535 _mm_set_pd(double __w, double __x)
537 return (__m128d){ __x, __w };
540 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
541 _mm_setr_pd(double __w, double __x)
543 return (__m128d){ __w, __x };
546 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
549 return (__m128d){ 0, 0 };
552 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
553 _mm_move_sd(__m128d __a, __m128d __b)
555 return (__m128d){ __b[0], __a[1] };
558 static __inline__ void __attribute__((__always_inline__, __nodebug__))
559 _mm_store_sd(double *__dp, __m128d __a)
561 struct __mm_store_sd_struct {
563 } __attribute__((__packed__, __may_alias__));
564 ((struct __mm_store_sd_struct*)__dp)->__u = __a[0];
567 static __inline__ void __attribute__((__always_inline__, __nodebug__))
568 _mm_store1_pd(double *__dp, __m128d __a)
570 struct __mm_store1_pd_struct {
572 } __attribute__((__packed__, __may_alias__));
573 ((struct __mm_store1_pd_struct*)__dp)->__u[0] = __a[0];
574 ((struct __mm_store1_pd_struct*)__dp)->__u[1] = __a[0];
577 static __inline__ void __attribute__((__always_inline__, __nodebug__))
578 _mm_store_pd(double *__dp, __m128d __a)
580 *(__m128d *)__dp = __a;
583 static __inline__ void __attribute__((__always_inline__, __nodebug__))
584 _mm_storeu_pd(double *__dp, __m128d __a)
586 __builtin_ia32_storeupd(__dp, __a);
589 static __inline__ void __attribute__((__always_inline__, __nodebug__))
590 _mm_storer_pd(double *__dp, __m128d __a)
592 __a = __builtin_shufflevector(__a, __a, 1, 0);
593 *(__m128d *)__dp = __a;
596 static __inline__ void __attribute__((__always_inline__, __nodebug__))
597 _mm_storeh_pd(double *__dp, __m128d __a)
599 struct __mm_storeh_pd_struct {
601 } __attribute__((__packed__, __may_alias__));
602 ((struct __mm_storeh_pd_struct*)__dp)->__u = __a[1];
605 static __inline__ void __attribute__((__always_inline__, __nodebug__))
606 _mm_storel_pd(double *__dp, __m128d __a)
608 struct __mm_storeh_pd_struct {
610 } __attribute__((__packed__, __may_alias__));
611 ((struct __mm_storeh_pd_struct*)__dp)->__u = __a[0];
614 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
615 _mm_add_epi8(__m128i __a, __m128i __b)
617 return (__m128i)((__v16qi)__a + (__v16qi)__b);
620 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
621 _mm_add_epi16(__m128i __a, __m128i __b)
623 return (__m128i)((__v8hi)__a + (__v8hi)__b);
626 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
627 _mm_add_epi32(__m128i __a, __m128i __b)
629 return (__m128i)((__v4si)__a + (__v4si)__b);
632 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
633 _mm_add_si64(__m64 __a, __m64 __b)
638 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
639 _mm_add_epi64(__m128i __a, __m128i __b)
644 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
645 _mm_adds_epi8(__m128i __a, __m128i __b)
647 return (__m128i)__builtin_ia32_paddsb128((__v16qi)__a, (__v16qi)__b);
650 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
651 _mm_adds_epi16(__m128i __a, __m128i __b)
653 return (__m128i)__builtin_ia32_paddsw128((__v8hi)__a, (__v8hi)__b);
656 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
657 _mm_adds_epu8(__m128i __a, __m128i __b)
659 return (__m128i)__builtin_ia32_paddusb128((__v16qi)__a, (__v16qi)__b);
662 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
663 _mm_adds_epu16(__m128i __a, __m128i __b)
665 return (__m128i)__builtin_ia32_paddusw128((__v8hi)__a, (__v8hi)__b);
668 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
669 _mm_avg_epu8(__m128i __a, __m128i __b)
671 return (__m128i)__builtin_ia32_pavgb128((__v16qi)__a, (__v16qi)__b);
674 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
675 _mm_avg_epu16(__m128i __a, __m128i __b)
677 return (__m128i)__builtin_ia32_pavgw128((__v8hi)__a, (__v8hi)__b);
680 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
681 _mm_madd_epi16(__m128i __a, __m128i __b)
683 return (__m128i)__builtin_ia32_pmaddwd128((__v8hi)__a, (__v8hi)__b);
686 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
687 _mm_max_epi16(__m128i __a, __m128i __b)
689 return (__m128i)__builtin_ia32_pmaxsw128((__v8hi)__a, (__v8hi)__b);
692 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
693 _mm_max_epu8(__m128i __a, __m128i __b)
695 return (__m128i)__builtin_ia32_pmaxub128((__v16qi)__a, (__v16qi)__b);
698 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
699 _mm_min_epi16(__m128i __a, __m128i __b)
701 return (__m128i)__builtin_ia32_pminsw128((__v8hi)__a, (__v8hi)__b);
704 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
705 _mm_min_epu8(__m128i __a, __m128i __b)
707 return (__m128i)__builtin_ia32_pminub128((__v16qi)__a, (__v16qi)__b);
710 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
711 _mm_mulhi_epi16(__m128i __a, __m128i __b)
713 return (__m128i)__builtin_ia32_pmulhw128((__v8hi)__a, (__v8hi)__b);
716 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
717 _mm_mulhi_epu16(__m128i __a, __m128i __b)
719 return (__m128i)__builtin_ia32_pmulhuw128((__v8hi)__a, (__v8hi)__b);
722 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
723 _mm_mullo_epi16(__m128i __a, __m128i __b)
725 return (__m128i)((__v8hi)__a * (__v8hi)__b);
728 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
729 _mm_mul_su32(__m64 __a, __m64 __b)
731 return __builtin_ia32_pmuludq((__v2si)__a, (__v2si)__b);
734 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
735 _mm_mul_epu32(__m128i __a, __m128i __b)
737 return __builtin_ia32_pmuludq128((__v4si)__a, (__v4si)__b);
740 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
741 _mm_sad_epu8(__m128i __a, __m128i __b)
743 return __builtin_ia32_psadbw128((__v16qi)__a, (__v16qi)__b);
746 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
747 _mm_sub_epi8(__m128i __a, __m128i __b)
749 return (__m128i)((__v16qi)__a - (__v16qi)__b);
752 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
753 _mm_sub_epi16(__m128i __a, __m128i __b)
755 return (__m128i)((__v8hi)__a - (__v8hi)__b);
758 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
759 _mm_sub_epi32(__m128i __a, __m128i __b)
761 return (__m128i)((__v4si)__a - (__v4si)__b);
764 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
765 _mm_sub_si64(__m64 __a, __m64 __b)
770 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
771 _mm_sub_epi64(__m128i __a, __m128i __b)
776 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
777 _mm_subs_epi8(__m128i __a, __m128i __b)
779 return (__m128i)__builtin_ia32_psubsb128((__v16qi)__a, (__v16qi)__b);
782 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
783 _mm_subs_epi16(__m128i __a, __m128i __b)
785 return (__m128i)__builtin_ia32_psubsw128((__v8hi)__a, (__v8hi)__b);
788 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
789 _mm_subs_epu8(__m128i __a, __m128i __b)
791 return (__m128i)__builtin_ia32_psubusb128((__v16qi)__a, (__v16qi)__b);
794 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
795 _mm_subs_epu16(__m128i __a, __m128i __b)
797 return (__m128i)__builtin_ia32_psubusw128((__v8hi)__a, (__v8hi)__b);
800 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
801 _mm_and_si128(__m128i __a, __m128i __b)
806 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
807 _mm_andnot_si128(__m128i __a, __m128i __b)
812 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
813 _mm_or_si128(__m128i __a, __m128i __b)
818 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
819 _mm_xor_si128(__m128i __a, __m128i __b)
824 #define _mm_slli_si128(a, count) __extension__ ({ \
826 (__m128i)__builtin_ia32_pslldqi128(__a, (count)*8); })
828 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
829 _mm_slli_epi16(__m128i __a, int __count)
831 return (__m128i)__builtin_ia32_psllwi128((__v8hi)__a, __count);
834 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
835 _mm_sll_epi16(__m128i __a, __m128i __count)
837 return (__m128i)__builtin_ia32_psllw128((__v8hi)__a, (__v8hi)__count);
840 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
841 _mm_slli_epi32(__m128i __a, int __count)
843 return (__m128i)__builtin_ia32_pslldi128((__v4si)__a, __count);
846 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
847 _mm_sll_epi32(__m128i __a, __m128i __count)
849 return (__m128i)__builtin_ia32_pslld128((__v4si)__a, (__v4si)__count);
852 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
853 _mm_slli_epi64(__m128i __a, int __count)
855 return __builtin_ia32_psllqi128(__a, __count);
858 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
859 _mm_sll_epi64(__m128i __a, __m128i __count)
861 return __builtin_ia32_psllq128(__a, __count);
864 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
865 _mm_srai_epi16(__m128i __a, int __count)
867 return (__m128i)__builtin_ia32_psrawi128((__v8hi)__a, __count);
870 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
871 _mm_sra_epi16(__m128i __a, __m128i __count)
873 return (__m128i)__builtin_ia32_psraw128((__v8hi)__a, (__v8hi)__count);
876 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
877 _mm_srai_epi32(__m128i __a, int __count)
879 return (__m128i)__builtin_ia32_psradi128((__v4si)__a, __count);
882 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
883 _mm_sra_epi32(__m128i __a, __m128i __count)
885 return (__m128i)__builtin_ia32_psrad128((__v4si)__a, (__v4si)__count);
889 #define _mm_srli_si128(a, count) __extension__ ({ \
891 (__m128i)__builtin_ia32_psrldqi128(__a, (count)*8); })
893 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
894 _mm_srli_epi16(__m128i __a, int __count)
896 return (__m128i)__builtin_ia32_psrlwi128((__v8hi)__a, __count);
899 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
900 _mm_srl_epi16(__m128i __a, __m128i __count)
902 return (__m128i)__builtin_ia32_psrlw128((__v8hi)__a, (__v8hi)__count);
905 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
906 _mm_srli_epi32(__m128i __a, int __count)
908 return (__m128i)__builtin_ia32_psrldi128((__v4si)__a, __count);
911 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
912 _mm_srl_epi32(__m128i __a, __m128i __count)
914 return (__m128i)__builtin_ia32_psrld128((__v4si)__a, (__v4si)__count);
917 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
918 _mm_srli_epi64(__m128i __a, int __count)
920 return __builtin_ia32_psrlqi128(__a, __count);
923 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
924 _mm_srl_epi64(__m128i __a, __m128i __count)
926 return __builtin_ia32_psrlq128(__a, __count);
929 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
930 _mm_cmpeq_epi8(__m128i __a, __m128i __b)
932 return (__m128i)((__v16qi)__a == (__v16qi)__b);
935 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
936 _mm_cmpeq_epi16(__m128i __a, __m128i __b)
938 return (__m128i)((__v8hi)__a == (__v8hi)__b);
941 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
942 _mm_cmpeq_epi32(__m128i __a, __m128i __b)
944 return (__m128i)((__v4si)__a == (__v4si)__b);
947 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
948 _mm_cmpgt_epi8(__m128i __a, __m128i __b)
950 /* This function always performs a signed comparison, but __v16qi is a char
951 which may be signed or unsigned. */
952 typedef signed char __v16qs __attribute__((__vector_size__(16)));
953 return (__m128i)((__v16qs)__a > (__v16qs)__b);
956 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
957 _mm_cmpgt_epi16(__m128i __a, __m128i __b)
959 return (__m128i)((__v8hi)__a > (__v8hi)__b);
962 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
963 _mm_cmpgt_epi32(__m128i __a, __m128i __b)
965 return (__m128i)((__v4si)__a > (__v4si)__b);
968 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
969 _mm_cmplt_epi8(__m128i __a, __m128i __b)
971 return _mm_cmpgt_epi8(__b, __a);
974 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
975 _mm_cmplt_epi16(__m128i __a, __m128i __b)
977 return _mm_cmpgt_epi16(__b, __a);
980 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
981 _mm_cmplt_epi32(__m128i __a, __m128i __b)
983 return _mm_cmpgt_epi32(__b, __a);
987 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
988 _mm_cvtsi64_sd(__m128d __a, long long __b)
994 static __inline__ long long __attribute__((__always_inline__, __nodebug__))
995 _mm_cvtsd_si64(__m128d __a)
997 return __builtin_ia32_cvtsd2si64(__a);
1000 static __inline__ long long __attribute__((__always_inline__, __nodebug__))
1001 _mm_cvttsd_si64(__m128d __a)
1007 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
1008 _mm_cvtepi32_ps(__m128i __a)
1010 return __builtin_ia32_cvtdq2ps((__v4si)__a);
1013 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1014 _mm_cvtps_epi32(__m128 __a)
1016 return (__m128i)__builtin_ia32_cvtps2dq(__a);
1019 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1020 _mm_cvttps_epi32(__m128 __a)
1022 return (__m128i)__builtin_ia32_cvttps2dq(__a);
1025 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1026 _mm_cvtsi32_si128(int __a)
1028 return (__m128i)(__v4si){ __a, 0, 0, 0 };
1032 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1033 _mm_cvtsi64_si128(long long __a)
1035 return (__m128i){ __a, 0 };
1039 static __inline__ int __attribute__((__always_inline__, __nodebug__))
1040 _mm_cvtsi128_si32(__m128i __a)
1042 __v4si __b = (__v4si)__a;
1047 static __inline__ long long __attribute__((__always_inline__, __nodebug__))
1048 _mm_cvtsi128_si64(__m128i __a)
1054 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1055 _mm_load_si128(__m128i const *__p)
1060 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1061 _mm_loadu_si128(__m128i const *__p)
1063 struct __loadu_si128 {
1065 } __attribute__((packed, may_alias));
1066 return ((struct __loadu_si128*)__p)->__v;
1069 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1070 _mm_loadl_epi64(__m128i const *__p)
1072 struct __mm_loadl_epi64_struct {
1074 } __attribute__((__packed__, __may_alias__));
1075 return (__m128i) { ((struct __mm_loadl_epi64_struct*)__p)->__u, 0};
1078 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1079 _mm_set_epi64x(long long q1, long long q0)
1081 return (__m128i){ q0, q1 };
1084 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1085 _mm_set_epi64(__m64 q1, __m64 q0)
1087 return (__m128i){ (long long)q0, (long long)q1 };
1090 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1091 _mm_set_epi32(int i3, int i2, int i1, int i0)
1093 return (__m128i)(__v4si){ i0, i1, i2, i3};
1096 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1097 _mm_set_epi16(short w7, short w6, short w5, short w4, short w3, short w2, short w1, short w0)
1099 return (__m128i)(__v8hi){ w0, w1, w2, w3, w4, w5, w6, w7 };
1102 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1103 _mm_set_epi8(char b15, char b14, char b13, char b12, char b11, char b10, char b9, char b8, char b7, char b6, char b5, char b4, char b3, char b2, char b1, char b0)
1105 return (__m128i)(__v16qi){ b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 };
1108 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1109 _mm_set1_epi64x(long long __q)
1111 return (__m128i){ __q, __q };
1114 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1115 _mm_set1_epi64(__m64 __q)
1117 return (__m128i){ (long long)__q, (long long)__q };
1120 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1121 _mm_set1_epi32(int __i)
1123 return (__m128i)(__v4si){ __i, __i, __i, __i };
1126 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1127 _mm_set1_epi16(short __w)
1129 return (__m128i)(__v8hi){ __w, __w, __w, __w, __w, __w, __w, __w };
1132 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1133 _mm_set1_epi8(char __b)
1135 return (__m128i)(__v16qi){ __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b };
1138 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1139 _mm_setr_epi64(__m64 q0, __m64 q1)
1141 return (__m128i){ (long long)q0, (long long)q1 };
1144 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1145 _mm_setr_epi32(int i0, int i1, int i2, int i3)
1147 return (__m128i)(__v4si){ i0, i1, i2, i3};
1150 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1151 _mm_setr_epi16(short w0, short w1, short w2, short w3, short w4, short w5, short w6, short w7)
1153 return (__m128i)(__v8hi){ w0, w1, w2, w3, w4, w5, w6, w7 };
1156 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1157 _mm_setr_epi8(char b0, char b1, char b2, char b3, char b4, char b5, char b6, char b7, char b8, char b9, char b10, char b11, char b12, char b13, char b14, char b15)
1159 return (__m128i)(__v16qi){ b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 };
1162 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1163 _mm_setzero_si128(void)
1165 return (__m128i){ 0LL, 0LL };
1168 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1169 _mm_store_si128(__m128i *__p, __m128i __b)
1174 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1175 _mm_storeu_si128(__m128i *__p, __m128i __b)
1177 __builtin_ia32_storedqu((char *)__p, (__v16qi)__b);
1180 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1181 _mm_maskmoveu_si128(__m128i __d, __m128i __n, char *__p)
1183 __builtin_ia32_maskmovdqu((__v16qi)__d, (__v16qi)__n, __p);
1186 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1187 _mm_storel_epi64(__m128i *__p, __m128i __a)
1189 struct __mm_storel_epi64_struct {
1191 } __attribute__((__packed__, __may_alias__));
1192 ((struct __mm_storel_epi64_struct*)__p)->__u = __a[0];
1195 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1196 _mm_stream_pd(double *__p, __m128d __a)
1198 __builtin_ia32_movntpd(__p, __a);
1201 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1202 _mm_stream_si128(__m128i *__p, __m128i __a)
1204 __builtin_ia32_movntdq(__p, __a);
1207 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1208 _mm_stream_si32(int *__p, int __a)
1210 __builtin_ia32_movnti(__p, __a);
1213 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1214 _mm_clflush(void const *__p)
1216 __builtin_ia32_clflush(__p);
1219 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1222 __builtin_ia32_lfence();
1225 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1228 __builtin_ia32_mfence();
1231 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1232 _mm_packs_epi16(__m128i __a, __m128i __b)
1234 return (__m128i)__builtin_ia32_packsswb128((__v8hi)__a, (__v8hi)__b);
1237 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1238 _mm_packs_epi32(__m128i __a, __m128i __b)
1240 return (__m128i)__builtin_ia32_packssdw128((__v4si)__a, (__v4si)__b);
1243 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1244 _mm_packus_epi16(__m128i __a, __m128i __b)
1246 return (__m128i)__builtin_ia32_packuswb128((__v8hi)__a, (__v8hi)__b);
1249 static __inline__ int __attribute__((__always_inline__, __nodebug__))
1250 _mm_extract_epi16(__m128i __a, int __imm)
1252 __v8hi __b = (__v8hi)__a;
1253 return (unsigned short)__b[__imm];
1256 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1257 _mm_insert_epi16(__m128i __a, int __b, int __imm)
1259 __v8hi __c = (__v8hi)__a;
1260 __c[__imm & 7] = __b;
1261 return (__m128i)__c;
1264 static __inline__ int __attribute__((__always_inline__, __nodebug__))
1265 _mm_movemask_epi8(__m128i __a)
1267 return __builtin_ia32_pmovmskb128((__v16qi)__a);
1270 #define _mm_shuffle_epi32(a, imm) __extension__ ({ \
1271 __m128i __a = (a); \
1272 (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si) _mm_set1_epi32(0), \
1273 (imm) & 0x3, ((imm) & 0xc) >> 2, \
1274 ((imm) & 0x30) >> 4, ((imm) & 0xc0) >> 6); })
1276 #define _mm_shufflelo_epi16(a, imm) __extension__ ({ \
1277 __m128i __a = (a); \
1278 (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi) _mm_set1_epi16(0), \
1279 (imm) & 0x3, ((imm) & 0xc) >> 2, \
1280 ((imm) & 0x30) >> 4, ((imm) & 0xc0) >> 6, \
1283 #define _mm_shufflehi_epi16(a, imm) __extension__ ({ \
1284 __m128i __a = (a); \
1285 (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi) _mm_set1_epi16(0), \
1287 4 + (((imm) & 0x03) >> 0), \
1288 4 + (((imm) & 0x0c) >> 2), \
1289 4 + (((imm) & 0x30) >> 4), \
1290 4 + (((imm) & 0xc0) >> 6)); })
1292 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1293 _mm_unpackhi_epi8(__m128i __a, __m128i __b)
1295 return (__m128i)__builtin_shufflevector((__v16qi)__a, (__v16qi)__b, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15);
1298 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1299 _mm_unpackhi_epi16(__m128i __a, __m128i __b)
1301 return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7);
1304 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1305 _mm_unpackhi_epi32(__m128i __a, __m128i __b)
1307 return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 2, 4+2, 3, 4+3);
1310 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1311 _mm_unpackhi_epi64(__m128i __a, __m128i __b)
1313 return (__m128i)__builtin_shufflevector(__a, __b, 1, 2+1);
1316 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1317 _mm_unpacklo_epi8(__m128i __a, __m128i __b)
1319 return (__m128i)__builtin_shufflevector((__v16qi)__a, (__v16qi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7);
1322 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1323 _mm_unpacklo_epi16(__m128i __a, __m128i __b)
1325 return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3);
1328 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1329 _mm_unpacklo_epi32(__m128i __a, __m128i __b)
1331 return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 0, 4+0, 1, 4+1);
1334 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1335 _mm_unpacklo_epi64(__m128i __a, __m128i __b)
1337 return (__m128i)__builtin_shufflevector(__a, __b, 0, 2+0);
1340 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
1341 _mm_movepi64_pi64(__m128i __a)
1343 return (__m64)__a[0];
1346 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1347 _mm_movpi64_epi64(__m64 __a)
1349 return (__m128i){ (long long)__a, 0 };
1352 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1353 _mm_move_epi64(__m128i __a)
1355 return __builtin_shufflevector(__a, (__m128i){ 0 }, 0, 2);
1358 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
1359 _mm_unpackhi_pd(__m128d __a, __m128d __b)
1361 return __builtin_shufflevector(__a, __b, 1, 2+1);
1364 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
1365 _mm_unpacklo_pd(__m128d __a, __m128d __b)
1367 return __builtin_shufflevector(__a, __b, 0, 2+0);
1370 static __inline__ int __attribute__((__always_inline__, __nodebug__))
1371 _mm_movemask_pd(__m128d __a)
1373 return __builtin_ia32_movmskpd(__a);
1376 #define _mm_shuffle_pd(a, b, i) __extension__ ({ \
1377 __m128d __a = (a); \
1378 __m128d __b = (b); \
1379 __builtin_shufflevector(__a, __b, (i) & 1, (((i) & 2) >> 1) + 2); })
1381 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
1382 _mm_castpd_ps(__m128d __a)
1387 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1388 _mm_castpd_si128(__m128d __a)
1390 return (__m128i)__a;
1393 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
1394 _mm_castps_pd(__m128 __a)
1396 return (__m128d)__a;
1399 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
1400 _mm_castps_si128(__m128 __a)
1402 return (__m128i)__a;
1405 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
1406 _mm_castsi128_ps(__m128i __a)
1411 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
1412 _mm_castsi128_pd(__m128i __a)
1414 return (__m128d)__a;
1417 static __inline__ void __attribute__((__always_inline__, __nodebug__))
1420 __asm__ volatile ("pause");
1423 #define _MM_SHUFFLE2(x, y) (((x) << 1) | (y))
1425 #endif /* __SSE2__ */
1427 #endif /* __EMMINTRIN_H */