1 /*===---- xmmintrin.h - SSE intrinsics -------------------------------------===
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 *===-----------------------------------------------------------------------===
28 #error "SSE instruction set not enabled"
33 typedef int __v4si __attribute__((__vector_size__(16)));
34 typedef float __v4sf __attribute__((__vector_size__(16)));
35 typedef float __m128 __attribute__((__vector_size__(16)));
37 // This header should only be included in a hosted environment as it depends on
38 // a standard library to provide allocation routines.
40 #include <mm_malloc.h>
43 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
44 _mm_add_ss(__m128 a, __m128 b)
50 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
51 _mm_add_ps(__m128 a, __m128 b)
56 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
57 _mm_sub_ss(__m128 a, __m128 b)
63 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
64 _mm_sub_ps(__m128 a, __m128 b)
69 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
70 _mm_mul_ss(__m128 a, __m128 b)
76 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
77 _mm_mul_ps(__m128 a, __m128 b)
82 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
83 _mm_div_ss(__m128 a, __m128 b)
89 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
90 _mm_div_ps(__m128 a, __m128 b)
95 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
98 return __builtin_ia32_sqrtss(a);
101 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
102 _mm_sqrt_ps(__m128 a)
104 return __builtin_ia32_sqrtps(a);
107 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
110 return __builtin_ia32_rcpss(a);
113 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
116 return __builtin_ia32_rcpps(a);
119 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
120 _mm_rsqrt_ss(__m128 a)
122 return __builtin_ia32_rsqrtss(a);
125 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
126 _mm_rsqrt_ps(__m128 a)
128 return __builtin_ia32_rsqrtps(a);
131 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
132 _mm_min_ss(__m128 a, __m128 b)
134 return __builtin_ia32_minss(a, b);
137 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
138 _mm_min_ps(__m128 a, __m128 b)
140 return __builtin_ia32_minps(a, b);
143 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
144 _mm_max_ss(__m128 a, __m128 b)
146 return __builtin_ia32_maxss(a, b);
149 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
150 _mm_max_ps(__m128 a, __m128 b)
152 return __builtin_ia32_maxps(a, b);
155 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
156 _mm_and_ps(__m128 a, __m128 b)
158 return (__m128)((__v4si)a & (__v4si)b);
161 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
162 _mm_andnot_ps(__m128 a, __m128 b)
164 return (__m128)(~(__v4si)a & (__v4si)b);
167 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
168 _mm_or_ps(__m128 a, __m128 b)
170 return (__m128)((__v4si)a | (__v4si)b);
173 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
174 _mm_xor_ps(__m128 a, __m128 b)
176 return (__m128)((__v4si)a ^ (__v4si)b);
179 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
180 _mm_cmpeq_ss(__m128 a, __m128 b)
182 return (__m128)__builtin_ia32_cmpss(a, b, 0);
185 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
186 _mm_cmpeq_ps(__m128 a, __m128 b)
188 return (__m128)__builtin_ia32_cmpps(a, b, 0);
191 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
192 _mm_cmplt_ss(__m128 a, __m128 b)
194 return (__m128)__builtin_ia32_cmpss(a, b, 1);
197 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
198 _mm_cmplt_ps(__m128 a, __m128 b)
200 return (__m128)__builtin_ia32_cmpps(a, b, 1);
203 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
204 _mm_cmple_ss(__m128 a, __m128 b)
206 return (__m128)__builtin_ia32_cmpss(a, b, 2);
209 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
210 _mm_cmple_ps(__m128 a, __m128 b)
212 return (__m128)__builtin_ia32_cmpps(a, b, 2);
215 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
216 _mm_cmpgt_ss(__m128 a, __m128 b)
218 return (__m128)__builtin_ia32_cmpss(b, a, 1);
221 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
222 _mm_cmpgt_ps(__m128 a, __m128 b)
224 return (__m128)__builtin_ia32_cmpps(b, a, 1);
227 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
228 _mm_cmpge_ss(__m128 a, __m128 b)
230 return (__m128)__builtin_ia32_cmpss(b, a, 2);
233 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
234 _mm_cmpge_ps(__m128 a, __m128 b)
236 return (__m128)__builtin_ia32_cmpps(b, a, 2);
239 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
240 _mm_cmpneq_ss(__m128 a, __m128 b)
242 return (__m128)__builtin_ia32_cmpss(a, b, 4);
245 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
246 _mm_cmpneq_ps(__m128 a, __m128 b)
248 return (__m128)__builtin_ia32_cmpps(a, b, 4);
251 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
252 _mm_cmpnlt_ss(__m128 a, __m128 b)
254 return (__m128)__builtin_ia32_cmpss(a, b, 5);
257 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
258 _mm_cmpnlt_ps(__m128 a, __m128 b)
260 return (__m128)__builtin_ia32_cmpps(a, b, 5);
263 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
264 _mm_cmpnle_ss(__m128 a, __m128 b)
266 return (__m128)__builtin_ia32_cmpss(a, b, 6);
269 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
270 _mm_cmpnle_ps(__m128 a, __m128 b)
272 return (__m128)__builtin_ia32_cmpps(a, b, 6);
275 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
276 _mm_cmpngt_ss(__m128 a, __m128 b)
278 return (__m128)__builtin_ia32_cmpss(b, a, 5);
281 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
282 _mm_cmpngt_ps(__m128 a, __m128 b)
284 return (__m128)__builtin_ia32_cmpps(b, a, 5);
287 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
288 _mm_cmpnge_ss(__m128 a, __m128 b)
290 return (__m128)__builtin_ia32_cmpss(b, a, 6);
293 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
294 _mm_cmpnge_ps(__m128 a, __m128 b)
296 return (__m128)__builtin_ia32_cmpps(b, a, 6);
299 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
300 _mm_cmpord_ss(__m128 a, __m128 b)
302 return (__m128)__builtin_ia32_cmpss(a, b, 7);
305 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
306 _mm_cmpord_ps(__m128 a, __m128 b)
308 return (__m128)__builtin_ia32_cmpps(a, b, 7);
311 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
312 _mm_cmpunord_ss(__m128 a, __m128 b)
314 return (__m128)__builtin_ia32_cmpss(a, b, 3);
317 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
318 _mm_cmpunord_ps(__m128 a, __m128 b)
320 return (__m128)__builtin_ia32_cmpps(a, b, 3);
323 static __inline__ int __attribute__((__always_inline__, __nodebug__))
324 _mm_comieq_ss(__m128 a, __m128 b)
326 return __builtin_ia32_comieq(a, b);
329 static __inline__ int __attribute__((__always_inline__, __nodebug__))
330 _mm_comilt_ss(__m128 a, __m128 b)
332 return __builtin_ia32_comilt(a, b);
335 static __inline__ int __attribute__((__always_inline__, __nodebug__))
336 _mm_comile_ss(__m128 a, __m128 b)
338 return __builtin_ia32_comile(a, b);
341 static __inline__ int __attribute__((__always_inline__, __nodebug__))
342 _mm_comigt_ss(__m128 a, __m128 b)
344 return __builtin_ia32_comigt(a, b);
347 static __inline__ int __attribute__((__always_inline__, __nodebug__))
348 _mm_comige_ss(__m128 a, __m128 b)
350 return __builtin_ia32_comige(a, b);
353 static __inline__ int __attribute__((__always_inline__, __nodebug__))
354 _mm_comineq_ss(__m128 a, __m128 b)
356 return __builtin_ia32_comineq(a, b);
359 static __inline__ int __attribute__((__always_inline__, __nodebug__))
360 _mm_ucomieq_ss(__m128 a, __m128 b)
362 return __builtin_ia32_ucomieq(a, b);
365 static __inline__ int __attribute__((__always_inline__, __nodebug__))
366 _mm_ucomilt_ss(__m128 a, __m128 b)
368 return __builtin_ia32_ucomilt(a, b);
371 static __inline__ int __attribute__((__always_inline__, __nodebug__))
372 _mm_ucomile_ss(__m128 a, __m128 b)
374 return __builtin_ia32_ucomile(a, b);
377 static __inline__ int __attribute__((__always_inline__, __nodebug__))
378 _mm_ucomigt_ss(__m128 a, __m128 b)
380 return __builtin_ia32_ucomigt(a, b);
383 static __inline__ int __attribute__((__always_inline__, __nodebug__))
384 _mm_ucomige_ss(__m128 a, __m128 b)
386 return __builtin_ia32_ucomige(a, b);
389 static __inline__ int __attribute__((__always_inline__, __nodebug__))
390 _mm_ucomineq_ss(__m128 a, __m128 b)
392 return __builtin_ia32_ucomineq(a, b);
395 static __inline__ int __attribute__((__always_inline__, __nodebug__))
396 _mm_cvtss_si32(__m128 a)
398 return __builtin_ia32_cvtss2si(a);
401 static __inline__ int __attribute__((__always_inline__, __nodebug__))
402 _mm_cvt_ss2si(__m128 a)
404 return _mm_cvtss_si32(a);
409 static __inline__ long long __attribute__((__always_inline__, __nodebug__))
410 _mm_cvtss_si64(__m128 a)
412 return __builtin_ia32_cvtss2si64(a);
417 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
418 _mm_cvtps_pi32(__m128 a)
420 return (__m64)__builtin_ia32_cvtps2pi(a);
423 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
424 _mm_cvt_ps2pi(__m128 a)
426 return _mm_cvtps_pi32(a);
429 static __inline__ int __attribute__((__always_inline__, __nodebug__))
430 _mm_cvttss_si32(__m128 a)
435 static __inline__ int __attribute__((__always_inline__, __nodebug__))
436 _mm_cvtt_ss2si(__m128 a)
438 return _mm_cvttss_si32(a);
441 static __inline__ long long __attribute__((__always_inline__, __nodebug__))
442 _mm_cvttss_si64(__m128 a)
447 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
448 _mm_cvttps_pi32(__m128 a)
450 return (__m64)__builtin_ia32_cvttps2pi(a);
453 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
454 _mm_cvtt_ps2pi(__m128 a)
456 return _mm_cvttps_pi32(a);
459 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
460 _mm_cvtsi32_ss(__m128 a, int b)
466 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
467 _mm_cvt_si2ss(__m128 a, int b)
469 return _mm_cvtsi32_ss(a, b);
474 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
475 _mm_cvtsi64_ss(__m128 a, long long b)
483 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
484 _mm_cvtpi32_ps(__m128 a, __m64 b)
486 return __builtin_ia32_cvtpi2ps(a, (__v2si)b);
489 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
490 _mm_cvt_pi2ps(__m128 a, __m64 b)
492 return _mm_cvtpi32_ps(a, b);
495 static __inline__ float __attribute__((__always_inline__, __nodebug__))
496 _mm_cvtss_f32(__m128 a)
501 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
502 _mm_loadh_pi(__m128 a, const __m64 *p)
504 typedef float __mm_loadh_pi_v2f32 __attribute__((__vector_size__(8)));
505 struct __mm_loadh_pi_struct {
506 __mm_loadh_pi_v2f32 u;
507 } __attribute__((__packed__, __may_alias__));
508 __mm_loadh_pi_v2f32 b = ((struct __mm_loadh_pi_struct*)p)->u;
509 __m128 bb = __builtin_shufflevector(b, b, 0, 1, 0, 1);
510 return __builtin_shufflevector(a, bb, 0, 1, 4, 5);
513 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
514 _mm_loadl_pi(__m128 a, const __m64 *p)
516 typedef float __mm_loadl_pi_v2f32 __attribute__((__vector_size__(8)));
517 struct __mm_loadl_pi_struct {
518 __mm_loadl_pi_v2f32 u;
519 } __attribute__((__packed__, __may_alias__));
520 __mm_loadl_pi_v2f32 b = ((struct __mm_loadl_pi_struct*)p)->u;
521 __m128 bb = __builtin_shufflevector(b, b, 0, 1, 0, 1);
522 return __builtin_shufflevector(a, bb, 4, 5, 2, 3);
525 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
526 _mm_load_ss(const float *p)
528 struct __mm_load_ss_struct {
530 } __attribute__((__packed__, __may_alias__));
531 float u = ((struct __mm_load_ss_struct*)p)->u;
532 return (__m128){ u, 0, 0, 0 };
535 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
536 _mm_load1_ps(const float *p)
538 struct __mm_load1_ps_struct {
540 } __attribute__((__packed__, __may_alias__));
541 float u = ((struct __mm_load1_ps_struct*)p)->u;
542 return (__m128){ u, u, u, u };
545 #define _mm_load_ps1(p) _mm_load1_ps(p)
547 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
548 _mm_load_ps(const float *p)
553 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
554 _mm_loadu_ps(const float *p)
558 } __attribute__((__packed__, __may_alias__));
559 return ((struct __loadu_ps*)p)->v;
562 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
563 _mm_loadr_ps(const float *p)
565 __m128 a = _mm_load_ps(p);
566 return __builtin_shufflevector(a, a, 3, 2, 1, 0);
569 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
572 return (__m128){ w, 0, 0, 0 };
575 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
578 return (__m128){ w, w, w, w };
581 // Microsoft specific.
582 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
585 return _mm_set1_ps(w);
588 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
589 _mm_set_ps(float z, float y, float x, float w)
591 return (__m128){ w, x, y, z };
594 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
595 _mm_setr_ps(float z, float y, float x, float w)
597 return (__m128){ z, y, x, w };
600 static __inline__ __m128 __attribute__((__always_inline__))
603 return (__m128){ 0, 0, 0, 0 };
606 static __inline__ void __attribute__((__always_inline__))
607 _mm_storeh_pi(__m64 *p, __m128 a)
609 __builtin_ia32_storehps((__v2si *)p, a);
612 static __inline__ void __attribute__((__always_inline__))
613 _mm_storel_pi(__m64 *p, __m128 a)
615 __builtin_ia32_storelps((__v2si *)p, a);
618 static __inline__ void __attribute__((__always_inline__))
619 _mm_store_ss(float *p, __m128 a)
621 struct __mm_store_ss_struct {
623 } __attribute__((__packed__, __may_alias__));
624 ((struct __mm_store_ss_struct*)p)->u = a[0];
627 static __inline__ void __attribute__((__always_inline__, __nodebug__))
628 _mm_storeu_ps(float *p, __m128 a)
630 __builtin_ia32_storeups(p, a);
633 static __inline__ void __attribute__((__always_inline__, __nodebug__))
634 _mm_store1_ps(float *p, __m128 a)
636 a = __builtin_shufflevector(a, a, 0, 0, 0, 0);
640 static __inline__ void __attribute__((__always_inline__, __nodebug__))
641 _mm_store_ps1(float *p, __m128 a)
643 return _mm_store1_ps(p, a);
646 static __inline__ void __attribute__((__always_inline__, __nodebug__))
647 _mm_store_ps(float *p, __m128 a)
652 static __inline__ void __attribute__((__always_inline__, __nodebug__))
653 _mm_storer_ps(float *p, __m128 a)
655 a = __builtin_shufflevector(a, a, 3, 2, 1, 0);
659 #define _MM_HINT_T0 3
660 #define _MM_HINT_T1 2
661 #define _MM_HINT_T2 1
662 #define _MM_HINT_NTA 0
664 /* FIXME: We have to #define this because "sel" must be a constant integer, and
665 Sema doesn't do any form of constant propagation yet. */
667 #define _mm_prefetch(a, sel) (__builtin_prefetch((void *)(a), 0, sel))
669 static __inline__ void __attribute__((__always_inline__, __nodebug__))
670 _mm_stream_pi(__m64 *p, __m64 a)
672 __builtin_ia32_movntq(p, a);
675 static __inline__ void __attribute__((__always_inline__, __nodebug__))
676 _mm_stream_ps(float *p, __m128 a)
678 __builtin_ia32_movntps(p, a);
681 static __inline__ void __attribute__((__always_inline__, __nodebug__))
684 __builtin_ia32_sfence();
687 static __inline__ int __attribute__((__always_inline__, __nodebug__))
688 _mm_extract_pi16(__m64 a, int n)
690 __v4hi b = (__v4hi)a;
691 return (unsigned short)b[n & 3];
694 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
695 _mm_insert_pi16(__m64 a, int d, int n)
697 __v4hi b = (__v4hi)a;
702 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
703 _mm_max_pi16(__m64 a, __m64 b)
705 return (__m64)__builtin_ia32_pmaxsw((__v4hi)a, (__v4hi)b);
708 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
709 _mm_max_pu8(__m64 a, __m64 b)
711 return (__m64)__builtin_ia32_pmaxub((__v8qi)a, (__v8qi)b);
714 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
715 _mm_min_pi16(__m64 a, __m64 b)
717 return (__m64)__builtin_ia32_pminsw((__v4hi)a, (__v4hi)b);
720 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
721 _mm_min_pu8(__m64 a, __m64 b)
723 return (__m64)__builtin_ia32_pminub((__v8qi)a, (__v8qi)b);
726 static __inline__ int __attribute__((__always_inline__, __nodebug__))
727 _mm_movemask_pi8(__m64 a)
729 return __builtin_ia32_pmovmskb((__v8qi)a);
732 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
733 _mm_mulhi_pu16(__m64 a, __m64 b)
735 return (__m64)__builtin_ia32_pmulhuw((__v4hi)a, (__v4hi)b);
738 #define _mm_shuffle_pi16(a, n) \
739 ((__m64)__builtin_ia32_pshufw(a, n))
741 static __inline__ void __attribute__((__always_inline__, __nodebug__))
742 _mm_maskmove_si64(__m64 d, __m64 n, char *p)
744 __builtin_ia32_maskmovq((__v8qi)d, (__v8qi)n, p);
747 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
748 _mm_avg_pu8(__m64 a, __m64 b)
750 return (__m64)__builtin_ia32_pavgb((__v8qi)a, (__v8qi)b);
753 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
754 _mm_avg_pu16(__m64 a, __m64 b)
756 return (__m64)__builtin_ia32_pavgw((__v4hi)a, (__v4hi)b);
759 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
760 _mm_sad_pu8(__m64 a, __m64 b)
762 return (__m64)__builtin_ia32_psadbw((__v8qi)a, (__v8qi)b);
765 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
768 return __builtin_ia32_stmxcsr();
771 static __inline__ void __attribute__((__always_inline__, __nodebug__))
772 _mm_setcsr(unsigned int i)
774 __builtin_ia32_ldmxcsr(i);
777 #define _mm_shuffle_ps(a, b, mask) \
778 (__builtin_shufflevector((__v4sf)(a), (__v4sf)(b), \
779 (mask) & 0x3, ((mask) & 0xc) >> 2, \
780 (((mask) & 0x30) >> 4) + 4, \
781 (((mask) & 0xc0) >> 6) + 4))
783 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
784 _mm_unpackhi_ps(__m128 a, __m128 b)
786 return __builtin_shufflevector(a, b, 2, 6, 3, 7);
789 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
790 _mm_unpacklo_ps(__m128 a, __m128 b)
792 return __builtin_shufflevector(a, b, 0, 4, 1, 5);
795 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
796 _mm_move_ss(__m128 a, __m128 b)
798 return __builtin_shufflevector(a, b, 4, 1, 2, 3);
801 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
802 _mm_movehl_ps(__m128 a, __m128 b)
804 return __builtin_shufflevector(a, b, 6, 7, 2, 3);
807 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
808 _mm_movelh_ps(__m128 a, __m128 b)
810 return __builtin_shufflevector(a, b, 0, 1, 4, 5);
813 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
814 _mm_cvtpi16_ps(__m64 a)
819 b = _mm_setzero_si64();
820 b = _mm_cmpgt_pi16(b, a);
821 c = _mm_unpackhi_pi16(a, b);
822 r = _mm_setzero_ps();
823 r = _mm_cvtpi32_ps(r, c);
824 r = _mm_movelh_ps(r, r);
825 c = _mm_unpacklo_pi16(a, b);
826 r = _mm_cvtpi32_ps(r, c);
831 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
832 _mm_cvtpu16_ps(__m64 a)
837 b = _mm_setzero_si64();
838 c = _mm_unpackhi_pi16(a, b);
839 r = _mm_setzero_ps();
840 r = _mm_cvtpi32_ps(r, c);
841 r = _mm_movelh_ps(r, r);
842 c = _mm_unpacklo_pi16(a, b);
843 r = _mm_cvtpi32_ps(r, c);
848 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
849 _mm_cvtpi8_ps(__m64 a)
853 b = _mm_setzero_si64();
854 b = _mm_cmpgt_pi8(b, a);
855 b = _mm_unpacklo_pi8(a, b);
857 return _mm_cvtpi16_ps(b);
860 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
861 _mm_cvtpu8_ps(__m64 a)
865 b = _mm_setzero_si64();
866 b = _mm_unpacklo_pi8(a, b);
868 return _mm_cvtpi16_ps(b);
871 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
872 _mm_cvtpi32x2_ps(__m64 a, __m64 b)
876 c = _mm_setzero_ps();
877 c = _mm_cvtpi32_ps(c, b);
878 c = _mm_movelh_ps(c, c);
880 return _mm_cvtpi32_ps(c, a);
883 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
884 _mm_cvtps_pi16(__m128 a)
888 b = _mm_cvtps_pi32(a);
889 a = _mm_movehl_ps(a, a);
890 c = _mm_cvtps_pi32(a);
892 return _mm_packs_pi16(b, c);
895 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
896 _mm_cvtps_pi8(__m128 a)
900 b = _mm_cvtps_pi16(a);
901 c = _mm_setzero_si64();
903 return _mm_packs_pi16(b, c);
906 static __inline__ int __attribute__((__always_inline__, __nodebug__))
907 _mm_movemask_ps(__m128 a)
909 return __builtin_ia32_movmskps(a);
912 #define _MM_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w))
914 #define _MM_EXCEPT_INVALID (0x0001)
915 #define _MM_EXCEPT_DENORM (0x0002)
916 #define _MM_EXCEPT_DIV_ZERO (0x0004)
917 #define _MM_EXCEPT_OVERFLOW (0x0008)
918 #define _MM_EXCEPT_UNDERFLOW (0x0010)
919 #define _MM_EXCEPT_INEXACT (0x0020)
920 #define _MM_EXCEPT_MASK (0x003f)
922 #define _MM_MASK_INVALID (0x0080)
923 #define _MM_MASK_DENORM (0x0100)
924 #define _MM_MASK_DIV_ZERO (0x0200)
925 #define _MM_MASK_OVERFLOW (0x0400)
926 #define _MM_MASK_UNDERFLOW (0x0800)
927 #define _MM_MASK_INEXACT (0x1000)
928 #define _MM_MASK_MASK (0x1f80)
930 #define _MM_ROUND_NEAREST (0x0000)
931 #define _MM_ROUND_DOWN (0x2000)
932 #define _MM_ROUND_UP (0x4000)
933 #define _MM_ROUND_TOWARD_ZERO (0x6000)
934 #define _MM_ROUND_MASK (0x6000)
936 #define _MM_FLUSH_ZERO_MASK (0x8000)
937 #define _MM_FLUSH_ZERO_ON (0x8000)
938 #define _MM_FLUSH_ZERO_OFF (0x8000)
940 #define _MM_GET_EXCEPTION_MASK() (_mm_getcsr() & _MM_MASK_MASK)
941 #define _MM_GET_EXCEPTION_STATE() (_mm_getcsr() & _MM_EXCEPT_MASK)
942 #define _MM_GET_FLUSH_ZERO_MODE() (_mm_getcsr() & _MM_FLUSH_ZERO_MASK)
943 #define _MM_GET_ROUNDING_MODE() (_mm_getcsr() & _MM_ROUND_MASK)
945 #define _MM_SET_EXCEPTION_MASK(x) (_mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | (x)))
946 #define _MM_SET_EXCEPTION_STATE(x) (_mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | (x)))
947 #define _MM_SET_FLUSH_ZERO_MODE(x) (_mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | (x)))
948 #define _MM_SET_ROUNDING_MODE(x) (_mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | (x)))
950 #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
952 __m128 tmp3, tmp2, tmp1, tmp0; \
953 tmp0 = _mm_unpacklo_ps((row0), (row1)); \
954 tmp2 = _mm_unpacklo_ps((row2), (row3)); \
955 tmp1 = _mm_unpackhi_ps((row0), (row1)); \
956 tmp3 = _mm_unpackhi_ps((row2), (row3)); \
957 (row0) = _mm_movelh_ps(tmp0, tmp2); \
958 (row1) = _mm_movehl_ps(tmp2, tmp0); \
959 (row2) = _mm_movelh_ps(tmp1, tmp3); \
960 (row3) = _mm_movehl_ps(tmp3, tmp1); \
963 /* Aliases for compatibility. */
964 #define _m_pextrw _mm_extract_pi16
965 #define _m_pinsrw _mm_insert_pi16
966 #define _m_pmaxsw _mm_max_pi16
967 #define _m_pmaxub _mm_max_pu8
968 #define _m_pminsw _mm_min_pi16
969 #define _m_pminub _mm_min_pu8
970 #define _m_pmovmskb _mm_movemask_pi8
971 #define _m_pmulhuw _mm_mulhi_pu16
972 #define _m_pshufw _mm_shuffle_pi16
973 #define _m_maskmovq _mm_maskmove_si64
974 #define _m_pavgb _mm_avg_pu8
975 #define _m_pavgw _mm_avg_pu16
976 #define _m_psadbw _mm_sad_pu8
980 /* Ugly hack for backwards-compatibility (compatible with gcc) */
982 #include <emmintrin.h>
987 #endif /* __XMMINTRIN_H */