1 /*===---- xopintrin.h - XOP intrinsics -------------------------------------===
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 *===-----------------------------------------------------------------------===
25 #error "Never use <xopintrin.h> directly; include <x86intrin.h> instead."
32 # error "XOP instruction set is not enabled"
35 #include <fma4intrin.h>
37 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
38 _mm_maccs_epi16(__m128i __A, __m128i __B, __m128i __C)
40 return (__m128i)__builtin_ia32_vpmacssww((__v8hi)__A, (__v8hi)__B, (__v8hi)__C);
43 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
44 _mm_macc_epi16(__m128i __A, __m128i __B, __m128i __C)
46 return (__m128i)__builtin_ia32_vpmacsww((__v8hi)__A, (__v8hi)__B, (__v8hi)__C);
49 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
50 _mm_maccsd_epi16(__m128i __A, __m128i __B, __m128i __C)
52 return (__m128i)__builtin_ia32_vpmacsswd((__v8hi)__A, (__v8hi)__B, (__v4si)__C);
55 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
56 _mm_maccd_epi16(__m128i __A, __m128i __B, __m128i __C)
58 return (__m128i)__builtin_ia32_vpmacswd((__v8hi)__A, (__v8hi)__B, (__v4si)__C);
61 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
62 _mm_maccs_epi32(__m128i __A, __m128i __B, __m128i __C)
64 return (__m128i)__builtin_ia32_vpmacssdd((__v4si)__A, (__v4si)__B, (__v4si)__C);
67 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
68 _mm_macc_epi32(__m128i __A, __m128i __B, __m128i __C)
70 return (__m128i)__builtin_ia32_vpmacsdd((__v4si)__A, (__v4si)__B, (__v4si)__C);
73 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
74 _mm_maccslo_epi32(__m128i __A, __m128i __B, __m128i __C)
76 return (__m128i)__builtin_ia32_vpmacssdql((__v4si)__A, (__v4si)__B, (__v2di)__C);
79 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
80 _mm_macclo_epi32(__m128i __A, __m128i __B, __m128i __C)
82 return (__m128i)__builtin_ia32_vpmacsdql((__v4si)__A, (__v4si)__B, (__v2di)__C);
85 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
86 _mm_maccshi_epi32(__m128i __A, __m128i __B, __m128i __C)
88 return (__m128i)__builtin_ia32_vpmacssdqh((__v4si)__A, (__v4si)__B, (__v2di)__C);
91 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
92 _mm_macchi_epi32(__m128i __A, __m128i __B, __m128i __C)
94 return (__m128i)__builtin_ia32_vpmacsdqh((__v4si)__A, (__v4si)__B, (__v2di)__C);
97 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
98 _mm_maddsd_epi16(__m128i __A, __m128i __B, __m128i __C)
100 return (__m128i)__builtin_ia32_vpmadcsswd((__v8hi)__A, (__v8hi)__B, (__v4si)__C);
103 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
104 _mm_maddd_epi16(__m128i __A, __m128i __B, __m128i __C)
106 return (__m128i)__builtin_ia32_vpmadcswd((__v8hi)__A, (__v8hi)__B, (__v4si)__C);
109 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
110 _mm_haddw_epi8(__m128i __A)
112 return (__m128i)__builtin_ia32_vphaddbw((__v16qi)__A);
115 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
116 _mm_haddd_epi8(__m128i __A)
118 return (__m128i)__builtin_ia32_vphaddbd((__v16qi)__A);
121 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
122 _mm_haddq_epi8(__m128i __A)
124 return (__m128i)__builtin_ia32_vphaddbq((__v16qi)__A);
127 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
128 _mm_haddd_epi16(__m128i __A)
130 return (__m128i)__builtin_ia32_vphaddwd((__v8hi)__A);
133 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
134 _mm_haddq_epi16(__m128i __A)
136 return (__m128i)__builtin_ia32_vphaddwq((__v8hi)__A);
139 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
140 _mm_haddq_epi32(__m128i __A)
142 return (__m128i)__builtin_ia32_vphadddq((__v4si)__A);
145 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
146 _mm_haddw_epu8(__m128i __A)
148 return (__m128i)__builtin_ia32_vphaddubw((__v16qi)__A);
151 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
152 _mm_haddd_epu8(__m128i __A)
154 return (__m128i)__builtin_ia32_vphaddubd((__v16qi)__A);
157 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
158 _mm_haddq_epu8(__m128i __A)
160 return (__m128i)__builtin_ia32_vphaddubq((__v16qi)__A);
163 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
164 _mm_haddd_epu16(__m128i __A)
166 return (__m128i)__builtin_ia32_vphadduwd((__v8hi)__A);
169 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
170 _mm_haddq_epu16(__m128i __A)
172 return (__m128i)__builtin_ia32_vphadduwq((__v8hi)__A);
175 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
176 _mm_haddq_epu32(__m128i __A)
178 return (__m128i)__builtin_ia32_vphaddudq((__v4si)__A);
181 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
182 _mm_hsubw_epi8(__m128i __A)
184 return (__m128i)__builtin_ia32_vphsubbw((__v16qi)__A);
187 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
188 _mm_hsubd_epi16(__m128i __A)
190 return (__m128i)__builtin_ia32_vphsubwd((__v8hi)__A);
193 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
194 _mm_hsubq_epi32(__m128i __A)
196 return (__m128i)__builtin_ia32_vphsubdq((__v4si)__A);
199 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
200 _mm_cmov_si128(__m128i __A, __m128i __B, __m128i __C)
202 return (__m128i)__builtin_ia32_vpcmov(__A, __B, __C);
205 static __inline__ __m256i __attribute__((__always_inline__, __nodebug__))
206 _mm256_cmov_si256(__m256i __A, __m256i __B, __m256i __C)
208 return (__m256i)__builtin_ia32_vpcmov_256(__A, __B, __C);
211 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
212 _mm_perm_epi8(__m128i __A, __m128i __B, __m128i __C)
214 return (__m128i)__builtin_ia32_vpperm((__v16qi)__A, (__v16qi)__B, (__v16qi)__C);
217 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
218 _mm_rot_epi8(__m128i __A, __m128i __B)
220 return (__m128i)__builtin_ia32_vprotb((__v16qi)__A, (__v16qi)__B);
223 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
224 _mm_rot_epi16(__m128i __A, __m128i __B)
226 return (__m128i)__builtin_ia32_vprotw((__v8hi)__A, (__v8hi)__B);
229 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
230 _mm_rot_epi32(__m128i __A, __m128i __B)
232 return (__m128i)__builtin_ia32_vprotd((__v4si)__A, (__v4si)__B);
235 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
236 _mm_rot_epi64(__m128i __A, __m128i __B)
238 return (__m128i)__builtin_ia32_vprotq((__v2di)__A, (__v2di)__B);
241 #define _mm_roti_epi8(A, N) __extension__ ({ \
243 (__m128i)__builtin_ia32_vprotbi((__v16qi)__A, (N)); })
245 #define _mm_roti_epi16(A, N) __extension__ ({ \
247 (__m128i)__builtin_ia32_vprotwi((__v8hi)__A, (N)); })
249 #define _mm_roti_epi32(A, N) __extension__ ({ \
251 (__m128i)__builtin_ia32_vprotdi((__v4si)__A, (N)); })
253 #define _mm_roti_epi64(A, N) __extension__ ({ \
255 (__m128i)__builtin_ia32_vprotqi((__v2di)__A, (N)); })
257 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
258 _mm_shl_epi8(__m128i __A, __m128i __B)
260 return (__m128i)__builtin_ia32_vpshlb((__v16qi)__A, (__v16qi)__B);
263 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
264 _mm_shl_epi16(__m128i __A, __m128i __B)
266 return (__m128i)__builtin_ia32_vpshlw((__v8hi)__A, (__v8hi)__B);
269 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
270 _mm_shl_epi32(__m128i __A, __m128i __B)
272 return (__m128i)__builtin_ia32_vpshld((__v4si)__A, (__v4si)__B);
275 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
276 _mm_shl_epi64(__m128i __A, __m128i __B)
278 return (__m128i)__builtin_ia32_vpshlq((__v2di)__A, (__v2di)__B);
281 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
282 _mm_sha_epi8(__m128i __A, __m128i __B)
284 return (__m128i)__builtin_ia32_vpshab((__v16qi)__A, (__v16qi)__B);
287 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
288 _mm_sha_epi16(__m128i __A, __m128i __B)
290 return (__m128i)__builtin_ia32_vpshaw((__v8hi)__A, (__v8hi)__B);
293 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
294 _mm_sha_epi32(__m128i __A, __m128i __B)
296 return (__m128i)__builtin_ia32_vpshad((__v4si)__A, (__v4si)__B);
299 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
300 _mm_sha_epi64(__m128i __A, __m128i __B)
302 return (__m128i)__builtin_ia32_vpshaq((__v2di)__A, (__v2di)__B);
305 #define _mm_com_epu8(A, B, N) __extension__ ({ \
308 (__m128i)__builtin_ia32_vpcomub((__v16qi)__A, (__v16qi)__B, (N)); })
310 #define _mm_com_epu16(A, B, N) __extension__ ({ \
313 (__m128i)__builtin_ia32_vpcomuw((__v8hi)__A, (__v8hi)__B, (N)); })
315 #define _mm_com_epu32(A, B, N) __extension__ ({ \
318 (__m128i)__builtin_ia32_vpcomud((__v4si)__A, (__v4si)__B, (N)); })
320 #define _mm_com_epu64(A, B, N) __extension__ ({ \
323 (__m128i)__builtin_ia32_vpcomuq((__v2di)__A, (__v2di)__B, (N)); })
325 #define _mm_com_epi8(A, B, N) __extension__ ({ \
328 (__m128i)__builtin_ia32_vpcomb((__v16qi)__A, (__v16qi)__B, (N)); })
330 #define _mm_com_epi16(A, B, N) __extension__ ({ \
333 (__m128i)__builtin_ia32_vpcomw((__v8hi)__A, (__v8hi)__B, (N)); })
335 #define _mm_com_epi32(A, B, N) __extension__ ({ \
338 (__m128i)__builtin_ia32_vpcomd((__v4si)__A, (__v4si)__B, (N)); })
340 #define _mm_com_epi64(A, B, N) __extension__ ({ \
343 (__m128i)__builtin_ia32_vpcomq((__v2di)__A, (__v2di)__B, (N)); })
345 #define _mm_permute2_pd(X, Y, C, I) __extension__ ({ \
349 (__m128d)__builtin_ia32_vpermil2pd((__v2df)__X, (__v2df)__Y, \
350 (__v2di)__C, (I)); })
352 #define _mm256_permute2_pd(X, Y, C, I) __extension__ ({ \
356 (__m256d)__builtin_ia32_vpermil2pd256((__v4df)__X, (__v4df)__Y, \
357 (__v4di)__C, (I)); })
359 #define _mm_permute2_ps(X, Y, C, I) __extension__ ({ \
363 (__m128)__builtin_ia32_vpermil2ps((__v4sf)__X, (__v4sf)__Y, \
364 (__v4si)__C, (I)); })
366 #define _mm256_permute2_ps(X, Y, C, I) __extension__ ({ \
370 (__m256)__builtin_ia32_vpermil2ps256((__v8sf)__X, (__v8sf)__Y, \
371 (__v8si)__C, (I)); })
373 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
374 _mm_frcz_ss(__m128 __A)
376 return (__m128)__builtin_ia32_vfrczss((__v4sf)__A);
379 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
380 _mm_frcz_sd(__m128d __A)
382 return (__m128d)__builtin_ia32_vfrczsd((__v2df)__A);
385 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
386 _mm_frcz_ps(__m128 __A)
388 return (__m128)__builtin_ia32_vfrczps((__v4sf)__A);
391 static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
392 _mm_frcz_pd(__m128d __A)
394 return (__m128d)__builtin_ia32_vfrczpd((__v2df)__A);
397 static __inline__ __m256 __attribute__((__always_inline__, __nodebug__))
398 _mm256_frcz_ps(__m256 __A)
400 return (__m256)__builtin_ia32_vfrczps256((__v8sf)__A);
403 static __inline__ __m256d __attribute__((__always_inline__, __nodebug__))
404 _mm256_frcz_pd(__m256d __A)
406 return (__m256d)__builtin_ia32_vfrczpd256((__v4df)__A);
411 #endif /* __XOPINTRIN_H */