contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsX86.td

   1 //===- IntrinsicsX86.td - Defines X86 intrinsics -----------*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines all of the X86-specific intrinsics.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 //===----------------------------------------------------------------------===//
  14 // Interrupt traps
  15 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
  16   def int_x86_int : Intrinsic<[], [llvm_i8_ty], [ImmArg<0>]>;
  17 }
  18
  19 //===----------------------------------------------------------------------===//
  20 // SEH intrinsics for Windows
  21 let TargetPrefix = "x86" in {
  22   def int_x86_seh_lsda : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], [IntrNoMem]>;
  23
  24   // Marks the EH registration node created in LLVM IR prior to code generation.
  25   def int_x86_seh_ehregnode : Intrinsic<[], [llvm_ptr_ty], []>;
  26
  27   // Marks the EH guard slot node created in LLVM IR prior to code generation.
  28   def int_x86_seh_ehguard : Intrinsic<[], [llvm_ptr_ty], []>;
  29 }
  30
  31 //===----------------------------------------------------------------------===//
  32 // FLAGS.
  33 let TargetPrefix = "x86" in {
  34   def int_x86_flags_read_u32 : GCCBuiltin<"__builtin_ia32_readeflags_u32">,
  35         Intrinsic<[llvm_i32_ty], [], []>;
  36   def int_x86_flags_read_u64 : GCCBuiltin<"__builtin_ia32_readeflags_u64">,
  37         Intrinsic<[llvm_i64_ty], [], []>;
  38   def int_x86_flags_write_u32 : GCCBuiltin<"__builtin_ia32_writeeflags_u32">,
  39         Intrinsic<[], [llvm_i32_ty], []>;
  40   def int_x86_flags_write_u64 : GCCBuiltin<"__builtin_ia32_writeeflags_u64">,
  41         Intrinsic<[], [llvm_i64_ty], []>;
  42 }
  43
  44 //===----------------------------------------------------------------------===//
  45 // Read Time Stamp Counter.
  46 let TargetPrefix = "x86" in {
  47   def int_x86_rdtsc : GCCBuiltin<"__builtin_ia32_rdtsc">,
  48               Intrinsic<[llvm_i64_ty], [], []>;
  49   def int_x86_rdtscp :
  50               Intrinsic<[llvm_i64_ty, llvm_i32_ty], [], []>;
  51 }
  52
  53 // Read Performance-Monitoring Counter.
  54 let TargetPrefix = "x86" in {
  55   def int_x86_rdpmc : GCCBuiltin<"__builtin_ia32_rdpmc">,
  56               Intrinsic<[llvm_i64_ty], [llvm_i32_ty], []>;
  57 }
  58
  59 // Read processor ID.
  60 let TargetPrefix = "x86" in {
  61   def int_x86_rdpid : GCCBuiltin<"__builtin_ia32_rdpid">,
  62               Intrinsic<[llvm_i32_ty], [], []>;
  63 }
  64
  65 //===----------------------------------------------------------------------===//
  66 // CET SS
  67 let TargetPrefix = "x86" in {
  68   def int_x86_incsspd : GCCBuiltin<"__builtin_ia32_incsspd">,
  69               Intrinsic<[], [llvm_i32_ty], []>;
  70   def int_x86_incsspq : GCCBuiltin<"__builtin_ia32_incsspq">,
  71               Intrinsic<[], [llvm_i64_ty], []>;
  72   def int_x86_rdsspd : GCCBuiltin<"__builtin_ia32_rdsspd">,
  73               Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>;
  74   def int_x86_rdsspq : GCCBuiltin<"__builtin_ia32_rdsspq">,
  75               Intrinsic<[llvm_i64_ty], [llvm_i64_ty], []>;
  76   def int_x86_saveprevssp : GCCBuiltin<"__builtin_ia32_saveprevssp">,
  77               Intrinsic<[], [], []>;
  78   def int_x86_rstorssp : GCCBuiltin<"__builtin_ia32_rstorssp">,
  79               Intrinsic<[], [llvm_ptr_ty], []>;
  80   def int_x86_wrssd : GCCBuiltin<"__builtin_ia32_wrssd">,
  81               Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>;
  82   def int_x86_wrssq : GCCBuiltin<"__builtin_ia32_wrssq">,
  83               Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], []>;
  84   def int_x86_wrussd : GCCBuiltin<"__builtin_ia32_wrussd">,
  85               Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>;
  86   def int_x86_wrussq : GCCBuiltin<"__builtin_ia32_wrussq">,
  87               Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], []>;
  88   def int_x86_setssbsy : GCCBuiltin<"__builtin_ia32_setssbsy">,
  89               Intrinsic<[], [], []>;
  90   def int_x86_clrssbsy : GCCBuiltin<"__builtin_ia32_clrssbsy">,
  91               Intrinsic<[], [llvm_ptr_ty], []>;
  92 }
  93
  94 //===----------------------------------------------------------------------===//
  95 // 3DNow!
  96
  97 let TargetPrefix = "x86" in {
  98   def int_x86_3dnow_pavgusb : GCCBuiltin<"__builtin_ia32_pavgusb">,
  99               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 100                         [IntrNoMem]>;
 101   def int_x86_3dnow_pf2id : GCCBuiltin<"__builtin_ia32_pf2id">,
 102               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 103   def int_x86_3dnow_pfacc : GCCBuiltin<"__builtin_ia32_pfacc">,
 104               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 105                         [IntrNoMem]>;
 106   def int_x86_3dnow_pfadd : GCCBuiltin<"__builtin_ia32_pfadd">,
 107               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 108                         [IntrNoMem]>;
 109   def int_x86_3dnow_pfcmpeq : GCCBuiltin<"__builtin_ia32_pfcmpeq">,
 110               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 111                         [IntrNoMem]>;
 112   def int_x86_3dnow_pfcmpge : GCCBuiltin<"__builtin_ia32_pfcmpge">,
 113               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 114                         [IntrNoMem]>;
 115   def int_x86_3dnow_pfcmpgt : GCCBuiltin<"__builtin_ia32_pfcmpgt">,
 116               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 117                         [IntrNoMem]>;
 118   def int_x86_3dnow_pfmax : GCCBuiltin<"__builtin_ia32_pfmax">,
 119               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 120                         [IntrNoMem]>;
 121   def int_x86_3dnow_pfmin : GCCBuiltin<"__builtin_ia32_pfmin">,
 122               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 123                         [IntrNoMem]>;
 124   def int_x86_3dnow_pfmul : GCCBuiltin<"__builtin_ia32_pfmul">,
 125               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 126                         [IntrNoMem]>;
 127   def int_x86_3dnow_pfrcp : GCCBuiltin<"__builtin_ia32_pfrcp">,
 128               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 129   def int_x86_3dnow_pfrcpit1 : GCCBuiltin<"__builtin_ia32_pfrcpit1">,
 130               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 131                         [IntrNoMem]>;
 132   def int_x86_3dnow_pfrcpit2 : GCCBuiltin<"__builtin_ia32_pfrcpit2">,
 133               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 134                         [IntrNoMem]>;
 135   def int_x86_3dnow_pfrsqrt : GCCBuiltin<"__builtin_ia32_pfrsqrt">,
 136               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 137   def int_x86_3dnow_pfrsqit1 : GCCBuiltin<"__builtin_ia32_pfrsqit1">,
 138               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 139                         [IntrNoMem]>;
 140   def int_x86_3dnow_pfsub : GCCBuiltin<"__builtin_ia32_pfsub">,
 141               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 142                         [IntrNoMem]>;
 143   def int_x86_3dnow_pfsubr : GCCBuiltin<"__builtin_ia32_pfsubr">,
 144               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 145                         [IntrNoMem]>;
 146   def int_x86_3dnow_pi2fd : GCCBuiltin<"__builtin_ia32_pi2fd">,
 147               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 148   def int_x86_3dnow_pmulhrw : GCCBuiltin<"__builtin_ia32_pmulhrw">,
 149               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 150                         [IntrNoMem]>;
 151 }
 152
 153 //===----------------------------------------------------------------------===//
 154 // 3DNow! extensions
 155
 156 let TargetPrefix = "x86" in {
 157   def int_x86_3dnowa_pf2iw : GCCBuiltin<"__builtin_ia32_pf2iw">,
 158               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 159   def int_x86_3dnowa_pfnacc : GCCBuiltin<"__builtin_ia32_pfnacc">,
 160               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 161                         [IntrNoMem]>;
 162   def int_x86_3dnowa_pfpnacc : GCCBuiltin<"__builtin_ia32_pfpnacc">,
 163               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
 164                         [IntrNoMem]>;
 165   def int_x86_3dnowa_pi2fw : GCCBuiltin<"__builtin_ia32_pi2fw">,
 166               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 167   def int_x86_3dnowa_pswapd :
 168               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 169 }
 170
 171 //===----------------------------------------------------------------------===//
 172 // SSE1
 173
 174 // Arithmetic ops
 175 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 176   def int_x86_sse_rcp_ss : GCCBuiltin<"__builtin_ia32_rcpss">,
 177               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty],
 178                         [IntrNoMem]>;
 179   def int_x86_sse_rcp_ps : GCCBuiltin<"__builtin_ia32_rcpps">,
 180               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty],
 181                         [IntrNoMem]>;
 182   def int_x86_sse_rsqrt_ss : GCCBuiltin<"__builtin_ia32_rsqrtss">,
 183               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty],
 184                         [IntrNoMem]>;
 185   def int_x86_sse_rsqrt_ps : GCCBuiltin<"__builtin_ia32_rsqrtps">,
 186               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty],
 187                         [IntrNoMem]>;
 188   def int_x86_sse_min_ss : GCCBuiltin<"__builtin_ia32_minss">,
 189               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 190                          llvm_v4f32_ty], [IntrNoMem]>;
 191   def int_x86_sse_min_ps : GCCBuiltin<"__builtin_ia32_minps">,
 192               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 193                          llvm_v4f32_ty], [IntrNoMem]>;
 194   def int_x86_sse_max_ss : GCCBuiltin<"__builtin_ia32_maxss">,
 195               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 196                          llvm_v4f32_ty], [IntrNoMem]>;
 197   def int_x86_sse_max_ps : GCCBuiltin<"__builtin_ia32_maxps">,
 198               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 199                          llvm_v4f32_ty], [IntrNoMem]>;
 200 }
 201
 202 // Comparison ops
 203 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 204   def int_x86_sse_cmp_ss : GCCBuiltin<"__builtin_ia32_cmpss">,
 205               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 206                          llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
 207   // NOTE: This comparison intrinsic is not used by clang as long as the
 208   //       distinction in signaling behaviour is not implemented.
 209   def int_x86_sse_cmp_ps :
 210               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 211                          llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
 212   def int_x86_sse_comieq_ss : GCCBuiltin<"__builtin_ia32_comieq">,
 213               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 214                          llvm_v4f32_ty], [IntrNoMem]>;
 215   def int_x86_sse_comilt_ss : GCCBuiltin<"__builtin_ia32_comilt">,
 216               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 217                          llvm_v4f32_ty], [IntrNoMem]>;
 218   def int_x86_sse_comile_ss : GCCBuiltin<"__builtin_ia32_comile">,
 219               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 220                          llvm_v4f32_ty], [IntrNoMem]>;
 221   def int_x86_sse_comigt_ss : GCCBuiltin<"__builtin_ia32_comigt">,
 222               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 223                          llvm_v4f32_ty], [IntrNoMem]>;
 224   def int_x86_sse_comige_ss : GCCBuiltin<"__builtin_ia32_comige">,
 225               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 226                          llvm_v4f32_ty], [IntrNoMem]>;
 227   def int_x86_sse_comineq_ss : GCCBuiltin<"__builtin_ia32_comineq">,
 228               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 229                          llvm_v4f32_ty], [IntrNoMem]>;
 230   def int_x86_sse_ucomieq_ss : GCCBuiltin<"__builtin_ia32_ucomieq">,
 231               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 232                          llvm_v4f32_ty], [IntrNoMem]>;
 233   def int_x86_sse_ucomilt_ss : GCCBuiltin<"__builtin_ia32_ucomilt">,
 234               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 235                          llvm_v4f32_ty], [IntrNoMem]>;
 236   def int_x86_sse_ucomile_ss : GCCBuiltin<"__builtin_ia32_ucomile">,
 237               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 238                          llvm_v4f32_ty], [IntrNoMem]>;
 239   def int_x86_sse_ucomigt_ss : GCCBuiltin<"__builtin_ia32_ucomigt">,
 240               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 241                          llvm_v4f32_ty], [IntrNoMem]>;
 242   def int_x86_sse_ucomige_ss : GCCBuiltin<"__builtin_ia32_ucomige">,
 243               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 244                          llvm_v4f32_ty], [IntrNoMem]>;
 245   def int_x86_sse_ucomineq_ss : GCCBuiltin<"__builtin_ia32_ucomineq">,
 246               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
 247                          llvm_v4f32_ty], [IntrNoMem]>;
 248 }
 249
 250
 251 // Conversion ops
 252 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 253   def int_x86_sse_cvtss2si : GCCBuiltin<"__builtin_ia32_cvtss2si">,
 254               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 255   def int_x86_sse_cvtss2si64 : GCCBuiltin<"__builtin_ia32_cvtss2si64">,
 256               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 257   def int_x86_sse_cvttss2si : GCCBuiltin<"__builtin_ia32_cvttss2si">,
 258               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 259   def int_x86_sse_cvttss2si64 : GCCBuiltin<"__builtin_ia32_cvttss2si64">,
 260               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 261
 262   def int_x86_sse_cvtps2pi : GCCBuiltin<"__builtin_ia32_cvtps2pi">,
 263               Intrinsic<[llvm_x86mmx_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 264   def int_x86_sse_cvttps2pi: GCCBuiltin<"__builtin_ia32_cvttps2pi">,
 265               Intrinsic<[llvm_x86mmx_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 266   def int_x86_sse_cvtpi2ps : GCCBuiltin<"__builtin_ia32_cvtpi2ps">,
 267               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 268                          llvm_x86mmx_ty], [IntrNoMem]>;
 269 }
 270
 271 // Cacheability support ops
 272 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 273   def int_x86_sse_sfence : GCCBuiltin<"__builtin_ia32_sfence">,
 274               Intrinsic<[], [], []>;
 275 }
 276
 277 // Control register.
 278 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 279   def int_x86_sse_stmxcsr :
 280               Intrinsic<[], [llvm_ptr_ty],
 281                          [IntrWriteMem, IntrArgMemOnly,
 282                          // This prevents reordering with ldmxcsr
 283                          IntrHasSideEffects]>;
 284   def int_x86_sse_ldmxcsr :
 285               Intrinsic<[], [llvm_ptr_ty],
 286                         [IntrReadMem, IntrArgMemOnly, IntrHasSideEffects,
 287                          // FIXME: LDMXCSR does not actualy write to memory,
 288                          // but Fast and DAG Isel both use writing to memory
 289                          // as a proxy for having side effects.
 290                          IntrWriteMem]>;
 291 }
 292
 293 // Misc.
 294 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 295   def int_x86_sse_movmsk_ps : GCCBuiltin<"__builtin_ia32_movmskps">,
 296               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 297 }
 298
 299 //===----------------------------------------------------------------------===//
 300 // SSE2
 301
 302 // FP arithmetic ops
 303 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 304   def int_x86_sse2_min_sd : GCCBuiltin<"__builtin_ia32_minsd">,
 305               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 306                          llvm_v2f64_ty], [IntrNoMem]>;
 307   def int_x86_sse2_min_pd : GCCBuiltin<"__builtin_ia32_minpd">,
 308               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 309                          llvm_v2f64_ty], [IntrNoMem]>;
 310   def int_x86_sse2_max_sd : GCCBuiltin<"__builtin_ia32_maxsd">,
 311               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 312                          llvm_v2f64_ty], [IntrNoMem]>;
 313   def int_x86_sse2_max_pd : GCCBuiltin<"__builtin_ia32_maxpd">,
 314               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 315                          llvm_v2f64_ty], [IntrNoMem]>;
 316 }
 317
 318 // FP comparison ops
 319 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 320   def int_x86_sse2_cmp_sd : GCCBuiltin<"__builtin_ia32_cmpsd">,
 321               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 322                          llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
 323   // NOTE: This comparison intrinsic is not used by clang as long as the
 324   //       distinction in signaling behaviour is not implemented.
 325   def int_x86_sse2_cmp_pd :
 326               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 327                          llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
 328   def int_x86_sse2_comieq_sd : GCCBuiltin<"__builtin_ia32_comisdeq">,
 329               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 330                          llvm_v2f64_ty], [IntrNoMem]>;
 331   def int_x86_sse2_comilt_sd : GCCBuiltin<"__builtin_ia32_comisdlt">,
 332               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 333                          llvm_v2f64_ty], [IntrNoMem]>;
 334   def int_x86_sse2_comile_sd : GCCBuiltin<"__builtin_ia32_comisdle">,
 335               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 336                          llvm_v2f64_ty], [IntrNoMem]>;
 337   def int_x86_sse2_comigt_sd : GCCBuiltin<"__builtin_ia32_comisdgt">,
 338               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 339                          llvm_v2f64_ty], [IntrNoMem]>;
 340   def int_x86_sse2_comige_sd : GCCBuiltin<"__builtin_ia32_comisdge">,
 341               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 342                          llvm_v2f64_ty], [IntrNoMem]>;
 343   def int_x86_sse2_comineq_sd : GCCBuiltin<"__builtin_ia32_comisdneq">,
 344               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 345                          llvm_v2f64_ty], [IntrNoMem]>;
 346   def int_x86_sse2_ucomieq_sd : GCCBuiltin<"__builtin_ia32_ucomisdeq">,
 347               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 348                          llvm_v2f64_ty], [IntrNoMem]>;
 349   def int_x86_sse2_ucomilt_sd : GCCBuiltin<"__builtin_ia32_ucomisdlt">,
 350               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 351                          llvm_v2f64_ty], [IntrNoMem]>;
 352   def int_x86_sse2_ucomile_sd : GCCBuiltin<"__builtin_ia32_ucomisdle">,
 353               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 354                          llvm_v2f64_ty], [IntrNoMem]>;
 355   def int_x86_sse2_ucomigt_sd : GCCBuiltin<"__builtin_ia32_ucomisdgt">,
 356               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 357                          llvm_v2f64_ty], [IntrNoMem]>;
 358   def int_x86_sse2_ucomige_sd : GCCBuiltin<"__builtin_ia32_ucomisdge">,
 359               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 360                          llvm_v2f64_ty], [IntrNoMem]>;
 361   def int_x86_sse2_ucomineq_sd : GCCBuiltin<"__builtin_ia32_ucomisdneq">,
 362               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
 363                          llvm_v2f64_ty], [IntrNoMem]>;
 364 }
 365
 366 // Integer arithmetic ops.
 367 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 368   def int_x86_sse2_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw128">,
 369               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 370                          llvm_v8i16_ty], [IntrNoMem, Commutative]>;
 371   def int_x86_sse2_pmulh_w : GCCBuiltin<"__builtin_ia32_pmulhw128">,
 372               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 373                          llvm_v8i16_ty], [IntrNoMem, Commutative]>;
 374   def int_x86_sse2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd128">,
 375               Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty,
 376                          llvm_v8i16_ty], [IntrNoMem, Commutative]>;
 377   def int_x86_sse2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb128">,
 378               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
 379                          llvm_v16i8_ty], [IntrNoMem, Commutative]>;
 380   def int_x86_sse2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw128">,
 381               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 382                          llvm_v8i16_ty], [IntrNoMem, Commutative]>;
 383   def int_x86_sse2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw128">,
 384               Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty,
 385                          llvm_v16i8_ty], [IntrNoMem, Commutative]>;
 386 }
 387
 388 // Integer shift ops.
 389 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 390   def int_x86_sse2_psll_w : GCCBuiltin<"__builtin_ia32_psllw128">,
 391               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 392                          llvm_v8i16_ty], [IntrNoMem]>;
 393   def int_x86_sse2_psll_d : GCCBuiltin<"__builtin_ia32_pslld128">,
 394               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 395                          llvm_v4i32_ty], [IntrNoMem]>;
 396   def int_x86_sse2_psll_q : GCCBuiltin<"__builtin_ia32_psllq128">,
 397               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
 398                          llvm_v2i64_ty], [IntrNoMem]>;
 399   def int_x86_sse2_psrl_w : GCCBuiltin<"__builtin_ia32_psrlw128">,
 400               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 401                          llvm_v8i16_ty], [IntrNoMem]>;
 402   def int_x86_sse2_psrl_d : GCCBuiltin<"__builtin_ia32_psrld128">,
 403               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 404                          llvm_v4i32_ty], [IntrNoMem]>;
 405   def int_x86_sse2_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq128">,
 406               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
 407                          llvm_v2i64_ty], [IntrNoMem]>;
 408   def int_x86_sse2_psra_w : GCCBuiltin<"__builtin_ia32_psraw128">,
 409               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 410                          llvm_v8i16_ty], [IntrNoMem]>;
 411   def int_x86_sse2_psra_d : GCCBuiltin<"__builtin_ia32_psrad128">,
 412               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 413                          llvm_v4i32_ty], [IntrNoMem]>;
 414
 415   // Oddly these don't require an immediate due to a gcc compatibility issue.
 416   def int_x86_sse2_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi128">,
 417               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 418                          llvm_i32_ty], [IntrNoMem]>;
 419   def int_x86_sse2_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi128">,
 420               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 421                          llvm_i32_ty], [IntrNoMem]>;
 422   def int_x86_sse2_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi128">,
 423               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
 424                          llvm_i32_ty], [IntrNoMem]>;
 425   def int_x86_sse2_psrli_w : GCCBuiltin<"__builtin_ia32_psrlwi128">,
 426               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 427                          llvm_i32_ty], [IntrNoMem]>;
 428   def int_x86_sse2_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi128">,
 429               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 430                          llvm_i32_ty], [IntrNoMem]>;
 431   def int_x86_sse2_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi128">,
 432               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
 433                          llvm_i32_ty], [IntrNoMem]>;
 434   def int_x86_sse2_psrai_w : GCCBuiltin<"__builtin_ia32_psrawi128">,
 435               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 436                          llvm_i32_ty], [IntrNoMem]>;
 437   def int_x86_sse2_psrai_d : GCCBuiltin<"__builtin_ia32_psradi128">,
 438               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 439                          llvm_i32_ty], [IntrNoMem]>;
 440 }
 441
 442 // Conversion ops
 443 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 444   def int_x86_sse2_cvtpd2dq : GCCBuiltin<"__builtin_ia32_cvtpd2dq">,
 445               Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 446   def int_x86_sse2_cvttpd2dq : GCCBuiltin<"__builtin_ia32_cvttpd2dq">,
 447               Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 448   def int_x86_sse2_cvtpd2ps : GCCBuiltin<"__builtin_ia32_cvtpd2ps">,
 449               Intrinsic<[llvm_v4f32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 450   def int_x86_sse2_cvtps2dq : GCCBuiltin<"__builtin_ia32_cvtps2dq">,
 451               Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 452   def int_x86_sse2_cvttps2dq : GCCBuiltin<"__builtin_ia32_cvttps2dq">,
 453               Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 454   def int_x86_sse2_cvtsd2si : GCCBuiltin<"__builtin_ia32_cvtsd2si">,
 455               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 456   def int_x86_sse2_cvtsd2si64 : GCCBuiltin<"__builtin_ia32_cvtsd2si64">,
 457               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 458   def int_x86_sse2_cvttsd2si : GCCBuiltin<"__builtin_ia32_cvttsd2si">,
 459               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 460   def int_x86_sse2_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_cvttsd2si64">,
 461               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 462   def int_x86_sse2_cvtsd2ss : GCCBuiltin<"__builtin_ia32_cvtsd2ss">,
 463               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 464                          llvm_v2f64_ty], [IntrNoMem]>;
 465   def int_x86_sse_cvtpd2pi : GCCBuiltin<"__builtin_ia32_cvtpd2pi">,
 466               Intrinsic<[llvm_x86mmx_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 467   def int_x86_sse_cvttpd2pi: GCCBuiltin<"__builtin_ia32_cvttpd2pi">,
 468               Intrinsic<[llvm_x86mmx_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 469   def int_x86_sse_cvtpi2pd : GCCBuiltin<"__builtin_ia32_cvtpi2pd">,
 470               Intrinsic<[llvm_v2f64_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 471 }
 472
 473 // Misc.
 474 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 475   def int_x86_sse2_packsswb_128 : GCCBuiltin<"__builtin_ia32_packsswb128">,
 476               Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty,
 477                          llvm_v8i16_ty], [IntrNoMem]>;
 478   def int_x86_sse2_packssdw_128 : GCCBuiltin<"__builtin_ia32_packssdw128">,
 479               Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty,
 480                          llvm_v4i32_ty], [IntrNoMem]>;
 481   def int_x86_sse2_packuswb_128 : GCCBuiltin<"__builtin_ia32_packuswb128">,
 482               Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty,
 483                          llvm_v8i16_ty], [IntrNoMem]>;
 484   def int_x86_sse2_movmsk_pd : GCCBuiltin<"__builtin_ia32_movmskpd">,
 485               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
 486   def int_x86_sse2_pmovmskb_128 : GCCBuiltin<"__builtin_ia32_pmovmskb128">,
 487               Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 488   def int_x86_sse2_maskmov_dqu : GCCBuiltin<"__builtin_ia32_maskmovdqu">,
 489               Intrinsic<[], [llvm_v16i8_ty,
 490                          llvm_v16i8_ty, llvm_ptr_ty], []>;
 491   def int_x86_sse2_clflush : GCCBuiltin<"__builtin_ia32_clflush">,
 492               Intrinsic<[], [llvm_ptr_ty], []>;
 493   def int_x86_sse2_lfence : GCCBuiltin<"__builtin_ia32_lfence">,
 494               Intrinsic<[], [], []>;
 495   def int_x86_sse2_mfence : GCCBuiltin<"__builtin_ia32_mfence">,
 496               Intrinsic<[], [], []>;
 497   def int_x86_sse2_pause : GCCBuiltin<"__builtin_ia32_pause">,
 498               Intrinsic<[], [], []>;
 499 }
 500
 501 //===----------------------------------------------------------------------===//
 502 // SSE3
 503
 504 // Addition / subtraction ops.
 505 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 506   def int_x86_sse3_addsub_ps : GCCBuiltin<"__builtin_ia32_addsubps">,
 507               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 508                          llvm_v4f32_ty], [IntrNoMem]>;
 509   def int_x86_sse3_addsub_pd : GCCBuiltin<"__builtin_ia32_addsubpd">,
 510               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 511                          llvm_v2f64_ty], [IntrNoMem]>;
 512 }
 513
 514 // Horizontal ops.
 515 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 516   def int_x86_sse3_hadd_ps : GCCBuiltin<"__builtin_ia32_haddps">,
 517               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 518                          llvm_v4f32_ty], [IntrNoMem]>;
 519   def int_x86_sse3_hadd_pd : GCCBuiltin<"__builtin_ia32_haddpd">,
 520               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 521                          llvm_v2f64_ty], [IntrNoMem]>;
 522   def int_x86_sse3_hsub_ps : GCCBuiltin<"__builtin_ia32_hsubps">,
 523               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 524                          llvm_v4f32_ty], [IntrNoMem]>;
 525   def int_x86_sse3_hsub_pd : GCCBuiltin<"__builtin_ia32_hsubpd">,
 526               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 527                          llvm_v2f64_ty], [IntrNoMem]>;
 528 }
 529
 530 // Specialized unaligned load.
 531 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 532   def int_x86_sse3_ldu_dq : GCCBuiltin<"__builtin_ia32_lddqu">,
 533               Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty], [IntrReadMem]>;
 534 }
 535
 536 // Thread synchronization ops.
 537 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 538   def int_x86_sse3_monitor : GCCBuiltin<"__builtin_ia32_monitor">,
 539               Intrinsic<[], [llvm_ptr_ty,
 540                          llvm_i32_ty, llvm_i32_ty], []>;
 541   def int_x86_sse3_mwait : GCCBuiltin<"__builtin_ia32_mwait">,
 542               Intrinsic<[], [llvm_i32_ty,
 543                          llvm_i32_ty], []>;
 544 }
 545
 546 //===----------------------------------------------------------------------===//
 547 // SSSE3
 548
 549 // Horizontal arithmetic ops
 550 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 551   def int_x86_ssse3_phadd_w         : GCCBuiltin<"__builtin_ia32_phaddw">,
 552               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 553                          llvm_x86mmx_ty], [IntrNoMem]>;
 554   def int_x86_ssse3_phadd_w_128     : GCCBuiltin<"__builtin_ia32_phaddw128">,
 555               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 556                          llvm_v8i16_ty], [IntrNoMem]>;
 557
 558   def int_x86_ssse3_phadd_d         : GCCBuiltin<"__builtin_ia32_phaddd">,
 559               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 560                          llvm_x86mmx_ty], [IntrNoMem]>;
 561   def int_x86_ssse3_phadd_d_128     : GCCBuiltin<"__builtin_ia32_phaddd128">,
 562               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 563                          llvm_v4i32_ty], [IntrNoMem]>;
 564
 565   def int_x86_ssse3_phadd_sw        : GCCBuiltin<"__builtin_ia32_phaddsw">,
 566               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 567                          llvm_x86mmx_ty], [IntrNoMem]>;
 568   def int_x86_ssse3_phadd_sw_128    : GCCBuiltin<"__builtin_ia32_phaddsw128">,
 569               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 570                          llvm_v8i16_ty], [IntrNoMem]>;
 571
 572   def int_x86_ssse3_phsub_w         : GCCBuiltin<"__builtin_ia32_phsubw">,
 573               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 574                          llvm_x86mmx_ty], [IntrNoMem]>;
 575   def int_x86_ssse3_phsub_w_128     : GCCBuiltin<"__builtin_ia32_phsubw128">,
 576               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 577                          llvm_v8i16_ty], [IntrNoMem]>;
 578
 579   def int_x86_ssse3_phsub_d         : GCCBuiltin<"__builtin_ia32_phsubd">,
 580               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 581                          llvm_x86mmx_ty], [IntrNoMem]>;
 582   def int_x86_ssse3_phsub_d_128     : GCCBuiltin<"__builtin_ia32_phsubd128">,
 583               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 584                          llvm_v4i32_ty], [IntrNoMem]>;
 585
 586   def int_x86_ssse3_phsub_sw        : GCCBuiltin<"__builtin_ia32_phsubsw">,
 587               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 588                          llvm_x86mmx_ty], [IntrNoMem]>;
 589   def int_x86_ssse3_phsub_sw_128    : GCCBuiltin<"__builtin_ia32_phsubsw128">,
 590               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 591                          llvm_v8i16_ty], [IntrNoMem]>;
 592
 593   def int_x86_ssse3_pmadd_ub_sw     : GCCBuiltin<"__builtin_ia32_pmaddubsw">,
 594               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 595                          llvm_x86mmx_ty], [IntrNoMem]>;
 596   def int_x86_ssse3_pmadd_ub_sw_128 : GCCBuiltin<"__builtin_ia32_pmaddubsw128">,
 597               Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty,
 598                          llvm_v16i8_ty], [IntrNoMem]>;
 599 }
 600
 601 // Packed multiply high with round and scale
 602 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 603   def int_x86_ssse3_pmul_hr_sw      : GCCBuiltin<"__builtin_ia32_pmulhrsw">,
 604               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 605                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
 606   def int_x86_ssse3_pmul_hr_sw_128  : GCCBuiltin<"__builtin_ia32_pmulhrsw128">,
 607               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 608                          llvm_v8i16_ty], [IntrNoMem, Commutative]>;
 609 }
 610
 611 // Shuffle ops
 612 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 613   def int_x86_ssse3_pshuf_b         : GCCBuiltin<"__builtin_ia32_pshufb">,
 614               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 615                          llvm_x86mmx_ty], [IntrNoMem]>;
 616   def int_x86_ssse3_pshuf_b_128     : GCCBuiltin<"__builtin_ia32_pshufb128">,
 617               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
 618                          llvm_v16i8_ty], [IntrNoMem]>;
 619   def int_x86_sse_pshuf_w           : GCCBuiltin<"__builtin_ia32_pshufw">,
 620               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i8_ty],
 621                          [IntrNoMem, ImmArg<1>]>;
 622 }
 623
 624 // Sign ops
 625 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 626   def int_x86_ssse3_psign_b         : GCCBuiltin<"__builtin_ia32_psignb">,
 627               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 628                          llvm_x86mmx_ty], [IntrNoMem]>;
 629   def int_x86_ssse3_psign_b_128     : GCCBuiltin<"__builtin_ia32_psignb128">,
 630               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
 631                          llvm_v16i8_ty], [IntrNoMem]>;
 632
 633   def int_x86_ssse3_psign_w         : GCCBuiltin<"__builtin_ia32_psignw">,
 634               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 635                          llvm_x86mmx_ty], [IntrNoMem]>;
 636   def int_x86_ssse3_psign_w_128     : GCCBuiltin<"__builtin_ia32_psignw128">,
 637               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
 638                          llvm_v8i16_ty], [IntrNoMem]>;
 639
 640   def int_x86_ssse3_psign_d         : GCCBuiltin<"__builtin_ia32_psignd">,
 641               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
 642                          llvm_x86mmx_ty], [IntrNoMem]>;
 643   def int_x86_ssse3_psign_d_128     : GCCBuiltin<"__builtin_ia32_psignd128">,
 644               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
 645                          llvm_v4i32_ty], [IntrNoMem]>;
 646 }
 647
 648 // Absolute value ops
 649 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 650   def int_x86_ssse3_pabs_b     : GCCBuiltin<"__builtin_ia32_pabsb">,
 651               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 652
 653   def int_x86_ssse3_pabs_w     : GCCBuiltin<"__builtin_ia32_pabsw">,
 654               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 655
 656   def int_x86_ssse3_pabs_d     : GCCBuiltin<"__builtin_ia32_pabsd">,
 657               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 658 }
 659
 660 //===----------------------------------------------------------------------===//
 661 // SSE4.1
 662
 663 // FP rounding ops
 664 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 665   def int_x86_sse41_round_ss        : GCCBuiltin<"__builtin_ia32_roundss">,
 666               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
 667                          llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
 668   def int_x86_sse41_round_ps        : GCCBuiltin<"__builtin_ia32_roundps">,
 669               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 670                          llvm_i32_ty], [IntrNoMem, ImmArg<1>]>;
 671   def int_x86_sse41_round_sd        : GCCBuiltin<"__builtin_ia32_roundsd">,
 672               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
 673                          llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
 674   def int_x86_sse41_round_pd        : GCCBuiltin<"__builtin_ia32_roundpd">,
 675               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 676                          llvm_i32_ty], [IntrNoMem, ImmArg<1>]>;
 677 }
 678
 679 // Vector min element
 680 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 681   def int_x86_sse41_phminposuw     : GCCBuiltin<"__builtin_ia32_phminposuw128">,
 682               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty],
 683                         [IntrNoMem]>;
 684 }
 685
 686 // Advanced Encryption Standard (AES) Instructions
 687 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 688   def int_x86_aesni_aesimc          : GCCBuiltin<"__builtin_ia32_aesimc128">,
 689               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty],
 690                         [IntrNoMem]>;
 691
 692   def int_x86_aesni_aesenc          : GCCBuiltin<"__builtin_ia32_aesenc128">,
 693               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 694                         [IntrNoMem]>;
 695   def int_x86_aesni_aesenc_256      : GCCBuiltin<"__builtin_ia32_aesenc256">,
 696               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
 697                         [IntrNoMem]>;
 698   def int_x86_aesni_aesenc_512      : GCCBuiltin<"__builtin_ia32_aesenc512">,
 699               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
 700                         [IntrNoMem]>;
 701
 702   def int_x86_aesni_aesenclast : GCCBuiltin<"__builtin_ia32_aesenclast128">,
 703               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 704                         [IntrNoMem]>;
 705   def int_x86_aesni_aesenclast_256 :
 706     GCCBuiltin<"__builtin_ia32_aesenclast256">,
 707     Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
 708               [IntrNoMem]>;
 709   def int_x86_aesni_aesenclast_512 :
 710     GCCBuiltin<"__builtin_ia32_aesenclast512">,
 711     Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
 712               [IntrNoMem]>;
 713
 714   def int_x86_aesni_aesdec          : GCCBuiltin<"__builtin_ia32_aesdec128">,
 715               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 716                         [IntrNoMem]>;
 717   def int_x86_aesni_aesdec_256      : GCCBuiltin<"__builtin_ia32_aesdec256">,
 718               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
 719                         [IntrNoMem]>;
 720   def int_x86_aesni_aesdec_512      : GCCBuiltin<"__builtin_ia32_aesdec512">,
 721               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
 722                         [IntrNoMem]>;
 723
 724   def int_x86_aesni_aesdeclast : GCCBuiltin<"__builtin_ia32_aesdeclast128">,
 725               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 726                         [IntrNoMem]>;
 727   def int_x86_aesni_aesdeclast_256 :
 728     GCCBuiltin<"__builtin_ia32_aesdeclast256">,
 729     Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
 730               [IntrNoMem]>;
 731   def int_x86_aesni_aesdeclast_512 :
 732     GCCBuiltin<"__builtin_ia32_aesdeclast512">,
 733     Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
 734               [IntrNoMem]>;
 735
 736   def int_x86_aesni_aeskeygenassist :
 737               GCCBuiltin<"__builtin_ia32_aeskeygenassist128">,
 738               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty],
 739                         [IntrNoMem, ImmArg<1>]>;
 740 }
 741
 742 // PCLMUL instructions
 743 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
 744   def int_x86_pclmulqdq : GCCBuiltin<"__builtin_ia32_pclmulqdq128">,
 745           Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty],
 746                     [IntrNoMem, ImmArg<2>]>;
 747   def int_x86_pclmulqdq_256 : GCCBuiltin<"__builtin_ia32_pclmulqdq256">,
 748           Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty],
 749                     [IntrNoMem, ImmArg<2>]>;
 750   def int_x86_pclmulqdq_512 : GCCBuiltin<"__builtin_ia32_pclmulqdq512">,
 751           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty],
 752                     [IntrNoMem, ImmArg<2>]>;
 753 }
 754
 755 // Vector pack
 756 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 757   def int_x86_sse41_packusdw        : GCCBuiltin<"__builtin_ia32_packusdw128">,
 758               Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
 759                         [IntrNoMem]>;
 760 }
 761
 762 // Vector insert
 763 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 764   def int_x86_sse41_insertps       : GCCBuiltin<"__builtin_ia32_insertps128">,
 765           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty],
 766                     [IntrNoMem, ImmArg<2>]>;
 767 }
 768
 769 // Vector blend
 770 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 771   def int_x86_sse41_pblendvb         : GCCBuiltin<"__builtin_ia32_pblendvb128">,
 772         Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_v16i8_ty],
 773                   [IntrNoMem]>;
 774   def int_x86_sse41_blendvpd         : GCCBuiltin<"__builtin_ia32_blendvpd">,
 775         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,llvm_v2f64_ty],
 776                   [IntrNoMem]>;
 777   def int_x86_sse41_blendvps         : GCCBuiltin<"__builtin_ia32_blendvps">,
 778         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,llvm_v4f32_ty],
 779                   [IntrNoMem]>;
 780 }
 781
 782 // Vector dot product
 783 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 784   def int_x86_sse41_dppd            : GCCBuiltin<"__builtin_ia32_dppd">,
 785           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty],
 786                     [IntrNoMem, Commutative, ImmArg<2>]>;
 787   def int_x86_sse41_dpps            : GCCBuiltin<"__builtin_ia32_dpps">,
 788           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty],
 789                     [IntrNoMem, Commutative, ImmArg<2>]>;
 790 }
 791
 792 // Vector sum of absolute differences
 793 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 794   def int_x86_sse41_mpsadbw         : GCCBuiltin<"__builtin_ia32_mpsadbw128">,
 795           Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_i8_ty],
 796                     [IntrNoMem, Commutative, ImmArg<2>]>;
 797 }
 798
 799 // Test instruction with bitwise comparison.
 800 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
 801   def int_x86_sse41_ptestz          : GCCBuiltin<"__builtin_ia32_ptestz128">,
 802           Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 803                     [IntrNoMem]>;
 804   def int_x86_sse41_ptestc          : GCCBuiltin<"__builtin_ia32_ptestc128">,
 805           Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 806                     [IntrNoMem]>;
 807   def int_x86_sse41_ptestnzc        : GCCBuiltin<"__builtin_ia32_ptestnzc128">,
 808           Intrinsic<[llvm_i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 809                     [IntrNoMem]>;
 810 }
 811
 812 //===----------------------------------------------------------------------===//
 813 // SSE4.2
 814
 815 // Miscellaneous
 816 // CRC Instruction
 817 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
 818   def int_x86_sse42_crc32_32_8       : GCCBuiltin<"__builtin_ia32_crc32qi">,
 819           Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i8_ty],
 820                     [IntrNoMem]>;
 821   def int_x86_sse42_crc32_32_16      : GCCBuiltin<"__builtin_ia32_crc32hi">,
 822           Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i16_ty],
 823                     [IntrNoMem]>;
 824   def int_x86_sse42_crc32_32_32      : GCCBuiltin<"__builtin_ia32_crc32si">,
 825           Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 826                     [IntrNoMem]>;
 827   def int_x86_sse42_crc32_64_64      : GCCBuiltin<"__builtin_ia32_crc32di">,
 828           Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty],
 829                     [IntrNoMem]>;
 830 }
 831
 832 // String/text processing ops.
 833 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
 834   def int_x86_sse42_pcmpistrm128  : GCCBuiltin<"__builtin_ia32_pcmpistrm128">,
 835     Intrinsic<[llvm_v16i8_ty],
 836         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 837         [IntrNoMem, ImmArg<2>]>;
 838   def int_x86_sse42_pcmpistri128  : GCCBuiltin<"__builtin_ia32_pcmpistri128">,
 839     Intrinsic<[llvm_i32_ty],
 840         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 841         [IntrNoMem, ImmArg<2>]>;
 842   def int_x86_sse42_pcmpistria128 : GCCBuiltin<"__builtin_ia32_pcmpistria128">,
 843     Intrinsic<[llvm_i32_ty],
 844         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 845         [IntrNoMem, ImmArg<2>]>;
 846   def int_x86_sse42_pcmpistric128 : GCCBuiltin<"__builtin_ia32_pcmpistric128">,
 847     Intrinsic<[llvm_i32_ty],
 848         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 849         [IntrNoMem, ImmArg<2>]>;
 850   def int_x86_sse42_pcmpistrio128 : GCCBuiltin<"__builtin_ia32_pcmpistrio128">,
 851     Intrinsic<[llvm_i32_ty],
 852         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 853         [IntrNoMem, ImmArg<2>]>;
 854   def int_x86_sse42_pcmpistris128 : GCCBuiltin<"__builtin_ia32_pcmpistris128">,
 855     Intrinsic<[llvm_i32_ty],
 856         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 857         [IntrNoMem, ImmArg<2>]>;
 858   def int_x86_sse42_pcmpistriz128 : GCCBuiltin<"__builtin_ia32_pcmpistriz128">,
 859     Intrinsic<[llvm_i32_ty],
 860         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
 861         [IntrNoMem, ImmArg<2>]>;
 862   def int_x86_sse42_pcmpestrm128  : GCCBuiltin<"__builtin_ia32_pcmpestrm128">,
 863     Intrinsic<[llvm_v16i8_ty],
 864         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 865          llvm_i8_ty],
 866         [IntrNoMem, ImmArg<4>]>;
 867   def int_x86_sse42_pcmpestri128  : GCCBuiltin<"__builtin_ia32_pcmpestri128">,
 868     Intrinsic<[llvm_i32_ty],
 869         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 870          llvm_i8_ty],
 871         [IntrNoMem, ImmArg<4>]>;
 872   def int_x86_sse42_pcmpestria128 : GCCBuiltin<"__builtin_ia32_pcmpestria128">,
 873     Intrinsic<[llvm_i32_ty],
 874         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 875          llvm_i8_ty],
 876         [IntrNoMem, ImmArg<4>]>;
 877   def int_x86_sse42_pcmpestric128 : GCCBuiltin<"__builtin_ia32_pcmpestric128">,
 878     Intrinsic<[llvm_i32_ty],
 879         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 880          llvm_i8_ty],
 881         [IntrNoMem, ImmArg<4>]>;
 882   def int_x86_sse42_pcmpestrio128 : GCCBuiltin<"__builtin_ia32_pcmpestrio128">,
 883     Intrinsic<[llvm_i32_ty],
 884         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 885          llvm_i8_ty],
 886         [IntrNoMem, ImmArg<4>]>;
 887   def int_x86_sse42_pcmpestris128 : GCCBuiltin<"__builtin_ia32_pcmpestris128">,
 888     Intrinsic<[llvm_i32_ty],
 889         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 890          llvm_i8_ty],
 891         [IntrNoMem, ImmArg<4>]>;
 892   def int_x86_sse42_pcmpestriz128 : GCCBuiltin<"__builtin_ia32_pcmpestriz128">,
 893     Intrinsic<[llvm_i32_ty],
 894         [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty,
 895          llvm_i8_ty],
 896         [IntrNoMem, ImmArg<4>]>;
 897 }
 898
 899 //===----------------------------------------------------------------------===//
 900 // SSE4A
 901
 902 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 903   def int_x86_sse4a_extrqi : GCCBuiltin<"__builtin_ia32_extrqi">,
 904     Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty, llvm_i8_ty],
 905               [IntrNoMem, ImmArg<1>, ImmArg<2>]>;
 906   def int_x86_sse4a_extrq  : GCCBuiltin<"__builtin_ia32_extrq">,
 907     Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v16i8_ty], [IntrNoMem]>;
 908
 909   def int_x86_sse4a_insertqi : GCCBuiltin<"__builtin_ia32_insertqi">,
 910     Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
 911                                 llvm_i8_ty, llvm_i8_ty],
 912               [IntrNoMem, ImmArg<2>, ImmArg<3>]>;
 913   def int_x86_sse4a_insertq  : GCCBuiltin<"__builtin_ia32_insertq">,
 914     Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>;
 915 }
 916
 917 //===----------------------------------------------------------------------===//
 918 // AVX
 919
 920 // Arithmetic ops
 921 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 922   def int_x86_avx_addsub_pd_256 : GCCBuiltin<"__builtin_ia32_addsubpd256">,
 923         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
 924                   llvm_v4f64_ty], [IntrNoMem]>;
 925   def int_x86_avx_addsub_ps_256 : GCCBuiltin<"__builtin_ia32_addsubps256">,
 926         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
 927                   llvm_v8f32_ty], [IntrNoMem]>;
 928   def int_x86_avx_max_pd_256 : GCCBuiltin<"__builtin_ia32_maxpd256">,
 929         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
 930                   llvm_v4f64_ty], [IntrNoMem]>;
 931   def int_x86_avx_max_ps_256 : GCCBuiltin<"__builtin_ia32_maxps256">,
 932         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
 933                   llvm_v8f32_ty], [IntrNoMem]>;
 934   def int_x86_avx_min_pd_256 : GCCBuiltin<"__builtin_ia32_minpd256">,
 935         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
 936                   llvm_v4f64_ty], [IntrNoMem]>;
 937   def int_x86_avx_min_ps_256 : GCCBuiltin<"__builtin_ia32_minps256">,
 938         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
 939                   llvm_v8f32_ty], [IntrNoMem]>;
 940
 941   def int_x86_avx_rsqrt_ps_256 : GCCBuiltin<"__builtin_ia32_rsqrtps256">,
 942         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
 943
 944   def int_x86_avx_rcp_ps_256 : GCCBuiltin<"__builtin_ia32_rcpps256">,
 945         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
 946
 947   def int_x86_avx_round_pd_256 : GCCBuiltin<"__builtin_ia32_roundpd256">,
 948         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
 949                   llvm_i32_ty], [IntrNoMem, ImmArg<1>]>;
 950   def int_x86_avx_round_ps_256 : GCCBuiltin<"__builtin_ia32_roundps256">,
 951         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
 952                   llvm_i32_ty], [IntrNoMem, ImmArg<1>]>;
 953 }
 954
 955 // Horizontal ops
 956 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 957   def int_x86_avx_hadd_pd_256 : GCCBuiltin<"__builtin_ia32_haddpd256">,
 958         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
 959                   llvm_v4f64_ty], [IntrNoMem]>;
 960   def int_x86_avx_hsub_ps_256 : GCCBuiltin<"__builtin_ia32_hsubps256">,
 961         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
 962                   llvm_v8f32_ty], [IntrNoMem]>;
 963   def int_x86_avx_hsub_pd_256 : GCCBuiltin<"__builtin_ia32_hsubpd256">,
 964         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
 965                   llvm_v4f64_ty], [IntrNoMem]>;
 966   def int_x86_avx_hadd_ps_256 : GCCBuiltin<"__builtin_ia32_haddps256">,
 967         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
 968                   llvm_v8f32_ty], [IntrNoMem]>;
 969 }
 970
 971 // Vector permutation
 972 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 973   def int_x86_avx_vpermilvar_pd : GCCBuiltin<"__builtin_ia32_vpermilvarpd">,
 974         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
 975                   llvm_v2i64_ty], [IntrNoMem]>;
 976   def int_x86_avx_vpermilvar_ps : GCCBuiltin<"__builtin_ia32_vpermilvarps">,
 977         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
 978                   llvm_v4i32_ty], [IntrNoMem]>;
 979
 980   def int_x86_avx_vpermilvar_pd_256 :
 981         GCCBuiltin<"__builtin_ia32_vpermilvarpd256">,
 982         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4i64_ty], [IntrNoMem]>;
 983   def int_x86_avx_vpermilvar_ps_256 :
 984         GCCBuiltin<"__builtin_ia32_vpermilvarps256">,
 985         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty], [IntrNoMem]>;
 986
 987   def int_x86_avx512_vpermi2var_d_128 :
 988        GCCBuiltin<"__builtin_ia32_vpermi2vard128">,
 989        Intrinsic<[llvm_v4i32_ty],
 990                  [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 991
 992   def int_x86_avx512_vpermi2var_d_256 :
 993         GCCBuiltin<"__builtin_ia32_vpermi2vard256">,
 994         Intrinsic<[llvm_v8i32_ty],
 995                   [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>;
 996
 997   def int_x86_avx512_vpermi2var_d_512 :
 998         GCCBuiltin<"__builtin_ia32_vpermi2vard512">,
 999         Intrinsic<[llvm_v16i32_ty],
1000                   [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty],
1001                   [IntrNoMem]>;
1002
1003   def int_x86_avx512_vpermi2var_hi_128 :
1004         GCCBuiltin<"__builtin_ia32_vpermi2varhi128">,
1005         Intrinsic<[llvm_v8i16_ty],
1006                   [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>;
1007
1008   def int_x86_avx512_vpermi2var_hi_256 :
1009         GCCBuiltin<"__builtin_ia32_vpermi2varhi256">,
1010         Intrinsic<[llvm_v16i16_ty],
1011                   [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty],
1012                   [IntrNoMem]>;
1013
1014   def int_x86_avx512_vpermi2var_hi_512 :
1015         GCCBuiltin<"__builtin_ia32_vpermi2varhi512">,
1016         Intrinsic<[llvm_v32i16_ty],
1017                   [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty],
1018                   [IntrNoMem]>;
1019
1020   def int_x86_avx512_vpermi2var_pd_128 :
1021         GCCBuiltin<"__builtin_ia32_vpermi2varpd128">,
1022         Intrinsic<[llvm_v2f64_ty],
1023                   [llvm_v2f64_ty, llvm_v2i64_ty, llvm_v2f64_ty], [IntrNoMem]>;
1024
1025   def int_x86_avx512_vpermi2var_pd_256 :
1026         GCCBuiltin<"__builtin_ia32_vpermi2varpd256">,
1027         Intrinsic<[llvm_v4f64_ty],
1028                   [llvm_v4f64_ty, llvm_v4i64_ty, llvm_v4f64_ty], [IntrNoMem]>;
1029
1030   def int_x86_avx512_vpermi2var_pd_512 :
1031         GCCBuiltin<"__builtin_ia32_vpermi2varpd512">,
1032         Intrinsic<[llvm_v8f64_ty],
1033                   [llvm_v8f64_ty, llvm_v8i64_ty, llvm_v8f64_ty], [IntrNoMem]>;
1034
1035   def int_x86_avx512_vpermi2var_ps_128 :
1036         GCCBuiltin<"__builtin_ia32_vpermi2varps128">,
1037         Intrinsic<[llvm_v4f32_ty],
1038                   [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4f32_ty], [IntrNoMem]>;
1039
1040   def int_x86_avx512_vpermi2var_ps_256 :
1041         GCCBuiltin<"__builtin_ia32_vpermi2varps256">,
1042         Intrinsic<[llvm_v8f32_ty],
1043                   [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8f32_ty], [IntrNoMem]>;
1044
1045   def int_x86_avx512_vpermi2var_ps_512 :
1046         GCCBuiltin<"__builtin_ia32_vpermi2varps512">,
1047         Intrinsic<[llvm_v16f32_ty],
1048                   [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16f32_ty],
1049                   [IntrNoMem]>;
1050
1051   def int_x86_avx512_vpermi2var_q_128 :
1052         GCCBuiltin<"__builtin_ia32_vpermi2varq128">,
1053         Intrinsic<[llvm_v2i64_ty],
1054                   [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>;
1055
1056   def int_x86_avx512_vpermi2var_q_256 :
1057         GCCBuiltin<"__builtin_ia32_vpermi2varq256">,
1058         Intrinsic<[llvm_v4i64_ty],
1059                   [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>;
1060
1061   def int_x86_avx512_vpermi2var_q_512 :
1062         GCCBuiltin<"__builtin_ia32_vpermi2varq512">,
1063         Intrinsic<[llvm_v8i64_ty],
1064                   [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty], [IntrNoMem]>;
1065
1066   def int_x86_avx512_vpermi2var_qi_128 :
1067         GCCBuiltin<"__builtin_ia32_vpermi2varqi128">,
1068         Intrinsic<[llvm_v16i8_ty],
1069                   [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
1070
1071   def int_x86_avx512_vpermi2var_qi_256 :
1072         GCCBuiltin<"__builtin_ia32_vpermi2varqi256">,
1073         Intrinsic<[llvm_v32i8_ty],
1074                   [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>;
1075
1076   def int_x86_avx512_vpermi2var_qi_512 :
1077         GCCBuiltin<"__builtin_ia32_vpermi2varqi512">,
1078         Intrinsic<[llvm_v64i8_ty],
1079                   [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>;
1080
1081   def int_x86_avx512_vpermilvar_pd_512 :
1082         GCCBuiltin<"__builtin_ia32_vpermilvarpd512">,
1083           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8i64_ty],
1084           [IntrNoMem]>;
1085
1086   def int_x86_avx512_vpermilvar_ps_512 :
1087         GCCBuiltin<"__builtin_ia32_vpermilvarps512">,
1088           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16i32_ty],
1089           [IntrNoMem]>;
1090
1091   def int_x86_avx512_pshuf_b_512 :
1092         GCCBuiltin<"__builtin_ia32_pshufb512">,
1093           Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty],
1094           [IntrNoMem]>;
1095
1096 }
1097
1098 // GFNI Instructions
1099 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1100   def int_x86_vgf2p8affineinvqb_128 :
1101          GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v16qi">,
1102           Intrinsic<[llvm_v16i8_ty],
1103           [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
1104           [IntrNoMem, ImmArg<2>]>;
1105   def int_x86_vgf2p8affineinvqb_256 :
1106          GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v32qi">,
1107           Intrinsic<[llvm_v32i8_ty],
1108           [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty],
1109           [IntrNoMem, ImmArg<2>]>;
1110   def int_x86_vgf2p8affineinvqb_512 :
1111          GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v64qi">,
1112           Intrinsic<[llvm_v64i8_ty],
1113           [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty],
1114           [IntrNoMem, ImmArg<2>]>;
1115
1116   def int_x86_vgf2p8affineqb_128 :
1117          GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v16qi">,
1118           Intrinsic<[llvm_v16i8_ty],
1119           [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
1120           [IntrNoMem, ImmArg<2>]>;
1121   def int_x86_vgf2p8affineqb_256 :
1122          GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v32qi">,
1123           Intrinsic<[llvm_v32i8_ty],
1124           [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty],
1125           [IntrNoMem, ImmArg<2>]>;
1126   def int_x86_vgf2p8affineqb_512 :
1127          GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v64qi">,
1128           Intrinsic<[llvm_v64i8_ty],
1129           [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty],
1130           [IntrNoMem, ImmArg<2>]>;
1131
1132   def int_x86_vgf2p8mulb_128     :
1133          GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v16qi">,
1134           Intrinsic<[llvm_v16i8_ty],
1135           [llvm_v16i8_ty, llvm_v16i8_ty],
1136           [IntrNoMem]>;
1137   def int_x86_vgf2p8mulb_256     :
1138          GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v32qi">,
1139           Intrinsic<[llvm_v32i8_ty],
1140           [llvm_v32i8_ty, llvm_v32i8_ty],
1141           [IntrNoMem]>;
1142   def int_x86_vgf2p8mulb_512     :
1143          GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v64qi">,
1144           Intrinsic<[llvm_v64i8_ty],
1145           [llvm_v64i8_ty, llvm_v64i8_ty],
1146           [IntrNoMem]>;
1147 }
1148
1149 // Vector blend
1150 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1151   def int_x86_avx_blendv_pd_256 : GCCBuiltin<"__builtin_ia32_blendvpd256">,
1152         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
1153                   llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>;
1154   def int_x86_avx_blendv_ps_256 : GCCBuiltin<"__builtin_ia32_blendvps256">,
1155         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
1156                   llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>;
1157 }
1158
1159 // Vector dot product
1160 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1161   def int_x86_avx_dp_ps_256 : GCCBuiltin<"__builtin_ia32_dpps256">,
1162         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
1163                   llvm_v8f32_ty, llvm_i8_ty],
1164                   [IntrNoMem, Commutative, ImmArg<2>]>;
1165 }
1166
1167 // Vector compare
1168 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1169   def int_x86_avx_cmp_pd_256 :
1170         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
1171                   llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
1172   def int_x86_avx_cmp_ps_256 :
1173         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
1174                   llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
1175 }
1176
1177 // Vector convert
1178 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1179   def int_x86_avx_cvt_pd2_ps_256 : GCCBuiltin<"__builtin_ia32_cvtpd2ps256">,
1180         Intrinsic<[llvm_v4f32_ty], [llvm_v4f64_ty], [IntrNoMem]>;
1181   def int_x86_avx_cvt_ps2dq_256 : GCCBuiltin<"__builtin_ia32_cvtps2dq256">,
1182         Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
1183   def int_x86_avx_cvtt_pd2dq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2dq256">,
1184         Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty], [IntrNoMem]>;
1185   def int_x86_avx_cvt_pd2dq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2dq256">,
1186         Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty], [IntrNoMem]>;
1187   def int_x86_avx_cvtt_ps2dq_256 : GCCBuiltin<"__builtin_ia32_cvttps2dq256">,
1188         Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
1189 }
1190
1191 // Vector bit test
1192 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1193   def int_x86_avx_vtestz_pd : GCCBuiltin<"__builtin_ia32_vtestzpd">,
1194         Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
1195                   llvm_v2f64_ty], [IntrNoMem]>;
1196   def int_x86_avx_vtestc_pd : GCCBuiltin<"__builtin_ia32_vtestcpd">,
1197         Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
1198                   llvm_v2f64_ty], [IntrNoMem]>;
1199   def int_x86_avx_vtestnzc_pd : GCCBuiltin<"__builtin_ia32_vtestnzcpd">,
1200         Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
1201                   llvm_v2f64_ty], [IntrNoMem]>;
1202   def int_x86_avx_vtestz_ps : GCCBuiltin<"__builtin_ia32_vtestzps">,
1203         Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
1204                   llvm_v4f32_ty], [IntrNoMem]>;
1205   def int_x86_avx_vtestc_ps : GCCBuiltin<"__builtin_ia32_vtestcps">,
1206         Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
1207                   llvm_v4f32_ty], [IntrNoMem]>;
1208   def int_x86_avx_vtestnzc_ps : GCCBuiltin<"__builtin_ia32_vtestnzcps">,
1209         Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
1210                   llvm_v4f32_ty], [IntrNoMem]>;
1211   def int_x86_avx_vtestz_pd_256 : GCCBuiltin<"__builtin_ia32_vtestzpd256">,
1212         Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty,
1213                   llvm_v4f64_ty], [IntrNoMem]>;
1214   def int_x86_avx_vtestc_pd_256 : GCCBuiltin<"__builtin_ia32_vtestcpd256">,
1215         Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty,
1216                   llvm_v4f64_ty], [IntrNoMem]>;
1217   def int_x86_avx_vtestnzc_pd_256 : GCCBuiltin<"__builtin_ia32_vtestnzcpd256">,
1218         Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty,
1219                   llvm_v4f64_ty], [IntrNoMem]>;
1220   def int_x86_avx_vtestz_ps_256 : GCCBuiltin<"__builtin_ia32_vtestzps256">,
1221         Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty,
1222                   llvm_v8f32_ty], [IntrNoMem]>;
1223   def int_x86_avx_vtestc_ps_256 : GCCBuiltin<"__builtin_ia32_vtestcps256">,
1224         Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty,
1225                   llvm_v8f32_ty], [IntrNoMem]>;
1226   def int_x86_avx_vtestnzc_ps_256 : GCCBuiltin<"__builtin_ia32_vtestnzcps256">,
1227         Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty,
1228                   llvm_v8f32_ty], [IntrNoMem]>;
1229   def int_x86_avx_ptestz_256 : GCCBuiltin<"__builtin_ia32_ptestz256">,
1230         Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty,
1231                   llvm_v4i64_ty], [IntrNoMem]>;
1232   def int_x86_avx_ptestc_256 : GCCBuiltin<"__builtin_ia32_ptestc256">,
1233         Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty,
1234                   llvm_v4i64_ty], [IntrNoMem]>;
1235   def int_x86_avx_ptestnzc_256 : GCCBuiltin<"__builtin_ia32_ptestnzc256">,
1236         Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty,
1237                   llvm_v4i64_ty], [IntrNoMem]>;
1238
1239   def int_x86_avx512_fpclass_pd_128 :
1240           Intrinsic<[llvm_v2i1_ty], [llvm_v2f64_ty, llvm_i32_ty],
1241           [IntrNoMem, ImmArg<1>]>;
1242   def int_x86_avx512_fpclass_pd_256 :
1243           Intrinsic<[llvm_v4i1_ty], [llvm_v4f64_ty, llvm_i32_ty],
1244           [IntrNoMem, ImmArg<1>]>;
1245   def int_x86_avx512_fpclass_pd_512 :
1246           Intrinsic<[llvm_v8i1_ty], [llvm_v8f64_ty, llvm_i32_ty],
1247           [IntrNoMem, ImmArg<1>]>;
1248   def int_x86_avx512_fpclass_ps_128 :
1249           Intrinsic<[llvm_v4i1_ty], [llvm_v4f32_ty, llvm_i32_ty],
1250           [IntrNoMem, ImmArg<1>]>;
1251   def int_x86_avx512_fpclass_ps_256 :
1252           Intrinsic<[llvm_v8i1_ty], [llvm_v8f32_ty, llvm_i32_ty],
1253           [IntrNoMem, ImmArg<1>]>;
1254   def int_x86_avx512_fpclass_ps_512 :
1255           Intrinsic<[llvm_v16i1_ty], [llvm_v16f32_ty, llvm_i32_ty],
1256           [IntrNoMem, ImmArg<1>]>;
1257   def int_x86_avx512_mask_fpclass_sd :
1258          GCCBuiltin<"__builtin_ia32_fpclasssd_mask">,
1259           Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty],
1260           [IntrNoMem, ImmArg<1>]>;
1261   def int_x86_avx512_mask_fpclass_ss :
1262          GCCBuiltin<"__builtin_ia32_fpclassss_mask">,
1263           Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty],
1264           [IntrNoMem, ImmArg<1>]>;
1265 }
1266
1267 // Vector extract sign mask
1268 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1269   def int_x86_avx_movmsk_pd_256 : GCCBuiltin<"__builtin_ia32_movmskpd256">,
1270         Intrinsic<[llvm_i32_ty], [llvm_v4f64_ty], [IntrNoMem]>;
1271   def int_x86_avx_movmsk_ps_256 : GCCBuiltin<"__builtin_ia32_movmskps256">,
1272         Intrinsic<[llvm_i32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
1273 }
1274
1275 // Vector zero
1276 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1277   def int_x86_avx_vzeroall : GCCBuiltin<"__builtin_ia32_vzeroall">,
1278         Intrinsic<[], [], []>;
1279   def int_x86_avx_vzeroupper : GCCBuiltin<"__builtin_ia32_vzeroupper">,
1280         Intrinsic<[], [], []>;
1281 }
1282
1283 // SIMD load ops
1284 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1285   def int_x86_avx_ldu_dq_256 : GCCBuiltin<"__builtin_ia32_lddqu256">,
1286         Intrinsic<[llvm_v32i8_ty], [llvm_ptr_ty], [IntrReadMem]>;
1287 }
1288
1289 // Conditional load ops
1290 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1291   def int_x86_avx_maskload_pd : GCCBuiltin<"__builtin_ia32_maskloadpd">,
1292         Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2i64_ty],
1293                   [IntrReadMem, IntrArgMemOnly]>;
1294   def int_x86_avx_maskload_ps : GCCBuiltin<"__builtin_ia32_maskloadps">,
1295         Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4i32_ty],
1296                   [IntrReadMem, IntrArgMemOnly]>;
1297   def int_x86_avx_maskload_pd_256 : GCCBuiltin<"__builtin_ia32_maskloadpd256">,
1298         Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4i64_ty],
1299                   [IntrReadMem, IntrArgMemOnly]>;
1300   def int_x86_avx_maskload_ps_256 : GCCBuiltin<"__builtin_ia32_maskloadps256">,
1301         Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8i32_ty],
1302                   [IntrReadMem, IntrArgMemOnly]>;
1303 }
1304
1305 // Conditional store ops
1306 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1307   def int_x86_avx_maskstore_pd : GCCBuiltin<"__builtin_ia32_maskstorepd">,
1308         Intrinsic<[], [llvm_ptr_ty,
1309                   llvm_v2i64_ty, llvm_v2f64_ty], [IntrArgMemOnly]>;
1310   def int_x86_avx_maskstore_ps : GCCBuiltin<"__builtin_ia32_maskstoreps">,
1311         Intrinsic<[], [llvm_ptr_ty,
1312                   llvm_v4i32_ty, llvm_v4f32_ty], [IntrArgMemOnly]>;
1313   def int_x86_avx_maskstore_pd_256 :
1314         GCCBuiltin<"__builtin_ia32_maskstorepd256">,
1315         Intrinsic<[], [llvm_ptr_ty,
1316                   llvm_v4i64_ty, llvm_v4f64_ty], [IntrArgMemOnly]>;
1317   def int_x86_avx_maskstore_ps_256 :
1318         GCCBuiltin<"__builtin_ia32_maskstoreps256">,
1319         Intrinsic<[], [llvm_ptr_ty,
1320                   llvm_v8i32_ty, llvm_v8f32_ty], [IntrArgMemOnly]>;
1321 }
1322
1323 // BITALG bits shuffle
1324 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1325   def int_x86_avx512_vpshufbitqmb_128 :
1326     Intrinsic<[llvm_v16i1_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
1327   def int_x86_avx512_vpshufbitqmb_256 :
1328     Intrinsic<[llvm_v32i1_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>;
1329   def int_x86_avx512_vpshufbitqmb_512 :
1330     Intrinsic<[llvm_v64i1_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>;
1331 }
1332
1333 //===----------------------------------------------------------------------===//
1334 // AVX2
1335
1336 // Integer arithmetic ops.
1337 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1338   def int_x86_avx2_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw256">,
1339               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1340                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
1341   def int_x86_avx2_pmulh_w : GCCBuiltin<"__builtin_ia32_pmulhw256">,
1342               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1343                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
1344   def int_x86_avx2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd256">,
1345               Intrinsic<[llvm_v8i32_ty], [llvm_v16i16_ty,
1346                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
1347   def int_x86_avx2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb256">,
1348               Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
1349                          llvm_v32i8_ty], [IntrNoMem, Commutative]>;
1350   def int_x86_avx2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw256">,
1351               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1352                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
1353   def int_x86_avx2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw256">,
1354               Intrinsic<[llvm_v4i64_ty], [llvm_v32i8_ty,
1355                          llvm_v32i8_ty], [IntrNoMem, Commutative]>;
1356 }
1357
1358 // Integer shift ops.
1359 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1360   def int_x86_avx2_psll_w : GCCBuiltin<"__builtin_ia32_psllw256">,
1361               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1362                          llvm_v8i16_ty], [IntrNoMem]>;
1363   def int_x86_avx2_psll_d : GCCBuiltin<"__builtin_ia32_pslld256">,
1364               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1365                          llvm_v4i32_ty], [IntrNoMem]>;
1366   def int_x86_avx2_psll_q : GCCBuiltin<"__builtin_ia32_psllq256">,
1367               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
1368                          llvm_v2i64_ty], [IntrNoMem]>;
1369   def int_x86_avx2_psrl_w : GCCBuiltin<"__builtin_ia32_psrlw256">,
1370               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1371                          llvm_v8i16_ty], [IntrNoMem]>;
1372   def int_x86_avx2_psrl_d : GCCBuiltin<"__builtin_ia32_psrld256">,
1373               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1374                          llvm_v4i32_ty], [IntrNoMem]>;
1375   def int_x86_avx2_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq256">,
1376               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
1377                          llvm_v2i64_ty], [IntrNoMem]>;
1378   def int_x86_avx2_psra_w : GCCBuiltin<"__builtin_ia32_psraw256">,
1379               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1380                          llvm_v8i16_ty], [IntrNoMem]>;
1381   def int_x86_avx2_psra_d : GCCBuiltin<"__builtin_ia32_psrad256">,
1382               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1383                          llvm_v4i32_ty], [IntrNoMem]>;
1384
1385   // Oddly these don't require an immediate due to a gcc compatibility issue.
1386   def int_x86_avx2_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi256">,
1387               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1388                          llvm_i32_ty], [IntrNoMem]>;
1389   def int_x86_avx2_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi256">,
1390               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1391                          llvm_i32_ty], [IntrNoMem]>;
1392   def int_x86_avx2_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi256">,
1393               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
1394                          llvm_i32_ty], [IntrNoMem]>;
1395   def int_x86_avx2_psrli_w : GCCBuiltin<"__builtin_ia32_psrlwi256">,
1396               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1397                          llvm_i32_ty], [IntrNoMem]>;
1398   def int_x86_avx2_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi256">,
1399               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1400                          llvm_i32_ty], [IntrNoMem]>;
1401   def int_x86_avx2_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi256">,
1402               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
1403                          llvm_i32_ty], [IntrNoMem]>;
1404   def int_x86_avx2_psrai_w : GCCBuiltin<"__builtin_ia32_psrawi256">,
1405               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1406                          llvm_i32_ty], [IntrNoMem]>;
1407   def int_x86_avx2_psrai_d : GCCBuiltin<"__builtin_ia32_psradi256">,
1408               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1409                          llvm_i32_ty], [IntrNoMem]>;
1410
1411   def int_x86_avx512_psra_q_128 : GCCBuiltin<"__builtin_ia32_psraq128">,
1412               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
1413                          llvm_v2i64_ty], [IntrNoMem]>;
1414   def int_x86_avx512_psra_q_256 : GCCBuiltin<"__builtin_ia32_psraq256">,
1415               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
1416                          llvm_v2i64_ty], [IntrNoMem]>;
1417
1418   // Oddly these don't require an immediate due to a gcc compatibility issue.
1419   def int_x86_avx512_psrai_q_128 : GCCBuiltin<"__builtin_ia32_psraqi128">,
1420               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
1421                          llvm_i32_ty], [IntrNoMem]>;
1422   def int_x86_avx512_psrai_q_256 : GCCBuiltin<"__builtin_ia32_psraqi256">,
1423               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
1424                          llvm_i32_ty], [IntrNoMem]>;
1425
1426   def int_x86_avx512_psll_w_512 : GCCBuiltin<"__builtin_ia32_psllw512">,
1427               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1428                          llvm_v8i16_ty], [IntrNoMem]>;
1429   def int_x86_avx512_psll_d_512 : GCCBuiltin<"__builtin_ia32_pslld512">,
1430               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
1431                          llvm_v4i32_ty], [IntrNoMem]>;
1432   def int_x86_avx512_psll_q_512 : GCCBuiltin<"__builtin_ia32_psllq512">,
1433               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
1434                          llvm_v2i64_ty], [IntrNoMem]>;
1435   def int_x86_avx512_psrl_w_512 : GCCBuiltin<"__builtin_ia32_psrlw512">,
1436               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1437                          llvm_v8i16_ty], [IntrNoMem]>;
1438   def int_x86_avx512_psrl_d_512 : GCCBuiltin<"__builtin_ia32_psrld512">,
1439               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
1440                          llvm_v4i32_ty], [IntrNoMem]>;
1441   def int_x86_avx512_psrl_q_512 : GCCBuiltin<"__builtin_ia32_psrlq512">,
1442               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
1443                          llvm_v2i64_ty], [IntrNoMem]>;
1444   def int_x86_avx512_psra_w_512 : GCCBuiltin<"__builtin_ia32_psraw512">,
1445               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1446                          llvm_v8i16_ty], [IntrNoMem]>;
1447   def int_x86_avx512_psra_d_512 : GCCBuiltin<"__builtin_ia32_psrad512">,
1448               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
1449                          llvm_v4i32_ty], [IntrNoMem]>;
1450   def int_x86_avx512_psra_q_512 : GCCBuiltin<"__builtin_ia32_psraq512">,
1451               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
1452                          llvm_v2i64_ty], [IntrNoMem]>;
1453
1454   // Oddly these don't require an immediate due to a gcc compatibility issue.
1455   def int_x86_avx512_pslli_w_512 : GCCBuiltin<"__builtin_ia32_psllwi512">,
1456               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1457                          llvm_i32_ty], [IntrNoMem]>;
1458   def int_x86_avx512_pslli_d_512 : GCCBuiltin<"__builtin_ia32_pslldi512">,
1459               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
1460                          llvm_i32_ty], [IntrNoMem]>;
1461   def int_x86_avx512_pslli_q_512 : GCCBuiltin<"__builtin_ia32_psllqi512">,
1462               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
1463                          llvm_i32_ty], [IntrNoMem]>;
1464   def int_x86_avx512_psrli_w_512 : GCCBuiltin<"__builtin_ia32_psrlwi512">,
1465               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1466                          llvm_i32_ty], [IntrNoMem]>;
1467   def int_x86_avx512_psrli_d_512 : GCCBuiltin<"__builtin_ia32_psrldi512">,
1468               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
1469                          llvm_i32_ty], [IntrNoMem]>;
1470   def int_x86_avx512_psrli_q_512 : GCCBuiltin<"__builtin_ia32_psrlqi512">,
1471               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
1472                          llvm_i32_ty], [IntrNoMem]>;
1473   def int_x86_avx512_psrai_w_512 : GCCBuiltin<"__builtin_ia32_psrawi512">,
1474               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1475                          llvm_i32_ty], [IntrNoMem]>;
1476   def int_x86_avx512_psrai_d_512 : GCCBuiltin<"__builtin_ia32_psradi512">,
1477               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
1478                          llvm_i32_ty], [IntrNoMem]>;
1479   def int_x86_avx512_psrai_q_512 : GCCBuiltin<"__builtin_ia32_psraqi512">,
1480               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
1481                          llvm_i32_ty], [IntrNoMem]>;
1482
1483   def int_x86_avx512_pmultishift_qb_128:
1484         GCCBuiltin<"__builtin_ia32_vpmultishiftqb128">,
1485         Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
1486   def int_x86_avx512_pmultishift_qb_256:
1487         GCCBuiltin<"__builtin_ia32_vpmultishiftqb256">,
1488         Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>;
1489   def int_x86_avx512_pmultishift_qb_512:
1490         GCCBuiltin<"__builtin_ia32_vpmultishiftqb512">,
1491         Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>;
1492 }
1493
1494 // Pack ops.
1495 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1496   def int_x86_avx2_packsswb : GCCBuiltin<"__builtin_ia32_packsswb256">,
1497               Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty,
1498                          llvm_v16i16_ty], [IntrNoMem]>;
1499   def int_x86_avx2_packssdw : GCCBuiltin<"__builtin_ia32_packssdw256">,
1500               Intrinsic<[llvm_v16i16_ty], [llvm_v8i32_ty,
1501                          llvm_v8i32_ty], [IntrNoMem]>;
1502   def int_x86_avx2_packuswb : GCCBuiltin<"__builtin_ia32_packuswb256">,
1503               Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty,
1504                          llvm_v16i16_ty], [IntrNoMem]>;
1505   def int_x86_avx2_packusdw : GCCBuiltin<"__builtin_ia32_packusdw256">,
1506               Intrinsic<[llvm_v16i16_ty], [llvm_v8i32_ty,
1507                          llvm_v8i32_ty], [IntrNoMem]>;
1508 }
1509
1510 // Horizontal arithmetic ops
1511 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1512   def int_x86_avx2_phadd_w : GCCBuiltin<"__builtin_ia32_phaddw256">,
1513               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1514                          llvm_v16i16_ty], [IntrNoMem]>;
1515   def int_x86_avx2_phadd_d : GCCBuiltin<"__builtin_ia32_phaddd256">,
1516               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1517                          llvm_v8i32_ty], [IntrNoMem]>;
1518   def int_x86_avx2_phadd_sw : GCCBuiltin<"__builtin_ia32_phaddsw256">,
1519               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1520                          llvm_v16i16_ty], [IntrNoMem]>;
1521   def int_x86_avx2_phsub_w : GCCBuiltin<"__builtin_ia32_phsubw256">,
1522               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1523                          llvm_v16i16_ty], [IntrNoMem]>;
1524   def int_x86_avx2_phsub_d : GCCBuiltin<"__builtin_ia32_phsubd256">,
1525               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1526                          llvm_v8i32_ty], [IntrNoMem]>;
1527   def int_x86_avx2_phsub_sw : GCCBuiltin<"__builtin_ia32_phsubsw256">,
1528               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1529                          llvm_v16i16_ty], [IntrNoMem]>;
1530   def int_x86_avx2_pmadd_ub_sw : GCCBuiltin<"__builtin_ia32_pmaddubsw256">,
1531               Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty,
1532                          llvm_v32i8_ty], [IntrNoMem]>;
1533 }
1534
1535 // Sign ops
1536 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1537   def int_x86_avx2_psign_b : GCCBuiltin<"__builtin_ia32_psignb256">,
1538               Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
1539                          llvm_v32i8_ty], [IntrNoMem]>;
1540   def int_x86_avx2_psign_w : GCCBuiltin<"__builtin_ia32_psignw256">,
1541               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1542                          llvm_v16i16_ty], [IntrNoMem]>;
1543   def int_x86_avx2_psign_d : GCCBuiltin<"__builtin_ia32_psignd256">,
1544               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
1545                          llvm_v8i32_ty], [IntrNoMem]>;
1546 }
1547
1548 // Packed multiply high with round and scale
1549 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1550   def int_x86_avx2_pmul_hr_sw : GCCBuiltin<"__builtin_ia32_pmulhrsw256">,
1551               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
1552                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
1553   def int_x86_avx512_pmul_hr_sw_512 : GCCBuiltin<"__builtin_ia32_pmulhrsw512">,
1554               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
1555                          llvm_v32i16_ty], [IntrNoMem, Commutative]>;
1556 }
1557
1558 // Vector blend
1559 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1560   def int_x86_avx2_pblendvb : GCCBuiltin<"__builtin_ia32_pblendvb256">,
1561               Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
1562                          llvm_v32i8_ty], [IntrNoMem]>;
1563 }
1564
1565
1566 // Vector permutation
1567 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1568   def int_x86_avx2_permd : GCCBuiltin<"__builtin_ia32_permvarsi256">,
1569               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty],
1570                         [IntrNoMem]>;
1571   def int_x86_avx2_permps : GCCBuiltin<"__builtin_ia32_permvarsf256">,
1572               Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty],
1573                         [IntrNoMem]>;
1574 }
1575
1576 // Conditional load ops
1577 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1578   def int_x86_avx2_maskload_d : GCCBuiltin<"__builtin_ia32_maskloadd">,
1579         Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_v4i32_ty],
1580                   [IntrReadMem, IntrArgMemOnly]>;
1581   def int_x86_avx2_maskload_q : GCCBuiltin<"__builtin_ia32_maskloadq">,
1582         Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_v2i64_ty],
1583                   [IntrReadMem, IntrArgMemOnly]>;
1584   def int_x86_avx2_maskload_d_256 : GCCBuiltin<"__builtin_ia32_maskloadd256">,
1585         Intrinsic<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_v8i32_ty],
1586                   [IntrReadMem, IntrArgMemOnly]>;
1587   def int_x86_avx2_maskload_q_256 : GCCBuiltin<"__builtin_ia32_maskloadq256">,
1588         Intrinsic<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_v4i64_ty],
1589                   [IntrReadMem, IntrArgMemOnly]>;
1590 }
1591
1592 // Conditional store ops
1593 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1594   def int_x86_avx2_maskstore_d : GCCBuiltin<"__builtin_ia32_maskstored">,
1595         Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty],
1596                   [IntrArgMemOnly]>;
1597   def int_x86_avx2_maskstore_q : GCCBuiltin<"__builtin_ia32_maskstoreq">,
1598         Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty],
1599                   [IntrArgMemOnly]>;
1600   def int_x86_avx2_maskstore_d_256 :
1601         GCCBuiltin<"__builtin_ia32_maskstored256">,
1602         Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty],
1603                   [IntrArgMemOnly]>;
1604   def int_x86_avx2_maskstore_q_256 :
1605         GCCBuiltin<"__builtin_ia32_maskstoreq256">,
1606         Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty],
1607                   [IntrArgMemOnly]>;
1608 }
1609
1610 // Variable bit shift ops
1611 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1612   def int_x86_avx2_psllv_d : GCCBuiltin<"__builtin_ia32_psllv4si">,
1613               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
1614                         [IntrNoMem]>;
1615   def int_x86_avx2_psllv_d_256 : GCCBuiltin<"__builtin_ia32_psllv8si">,
1616               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty],
1617                         [IntrNoMem]>;
1618   def int_x86_avx2_psllv_q : GCCBuiltin<"__builtin_ia32_psllv2di">,
1619               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
1620                         [IntrNoMem]>;
1621   def int_x86_avx2_psllv_q_256 : GCCBuiltin<"__builtin_ia32_psllv4di">,
1622               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
1623                         [IntrNoMem]>;
1624
1625   def int_x86_avx512_psllv_d_512 : GCCBuiltin<"__builtin_ia32_psllv16si">,
1626               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty],
1627                         [IntrNoMem]>;
1628   def int_x86_avx512_psllv_q_512 : GCCBuiltin<"__builtin_ia32_psllv8di">,
1629               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
1630                         [IntrNoMem]>;
1631
1632   def int_x86_avx2_psrlv_d : GCCBuiltin<"__builtin_ia32_psrlv4si">,
1633               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
1634                         [IntrNoMem]>;
1635   def int_x86_avx2_psrlv_d_256 : GCCBuiltin<"__builtin_ia32_psrlv8si">,
1636               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty],
1637                         [IntrNoMem]>;
1638   def int_x86_avx2_psrlv_q : GCCBuiltin<"__builtin_ia32_psrlv2di">,
1639               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
1640                         [IntrNoMem]>;
1641   def int_x86_avx2_psrlv_q_256 : GCCBuiltin<"__builtin_ia32_psrlv4di">,
1642               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
1643                         [IntrNoMem]>;
1644
1645   def int_x86_avx512_psrlv_d_512 : GCCBuiltin<"__builtin_ia32_psrlv16si">,
1646               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty],
1647                         [IntrNoMem]>;
1648   def int_x86_avx512_psrlv_q_512 : GCCBuiltin<"__builtin_ia32_psrlv8di">,
1649               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
1650                         [IntrNoMem]>;
1651
1652   def int_x86_avx2_psrav_d : GCCBuiltin<"__builtin_ia32_psrav4si">,
1653               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
1654                         [IntrNoMem]>;
1655   def int_x86_avx2_psrav_d_256 : GCCBuiltin<"__builtin_ia32_psrav8si">,
1656               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty],
1657                         [IntrNoMem]>;
1658
1659   def int_x86_avx512_psrav_d_512 : GCCBuiltin<"__builtin_ia32_psrav16si">,
1660               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty],
1661                         [IntrNoMem]>;
1662   def int_x86_avx512_psrav_q_128 : GCCBuiltin<"__builtin_ia32_psravq128">,
1663               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
1664                         [IntrNoMem]>;
1665   def int_x86_avx512_psrav_q_256 : GCCBuiltin<"__builtin_ia32_psravq256">,
1666               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty],
1667                         [IntrNoMem]>;
1668   def int_x86_avx512_psrav_q_512 : GCCBuiltin<"__builtin_ia32_psrav8di">,
1669               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty],
1670                         [IntrNoMem]>;
1671
1672   def int_x86_avx512_psllv_w_128 : GCCBuiltin<"__builtin_ia32_psllv8hi">,
1673               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
1674                         [IntrNoMem]>;
1675   def int_x86_avx512_psllv_w_256 : GCCBuiltin<"__builtin_ia32_psllv16hi">,
1676               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty],
1677                         [IntrNoMem]>;
1678   def int_x86_avx512_psllv_w_512 : GCCBuiltin<"__builtin_ia32_psllv32hi">,
1679               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
1680                         [IntrNoMem]>;
1681
1682   def int_x86_avx512_psrlv_w_128 : GCCBuiltin<"__builtin_ia32_psrlv8hi">,
1683               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
1684                         [IntrNoMem]>;
1685   def int_x86_avx512_psrlv_w_256 : GCCBuiltin<"__builtin_ia32_psrlv16hi">,
1686               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty],
1687                         [IntrNoMem]>;
1688   def int_x86_avx512_psrlv_w_512 : GCCBuiltin<"__builtin_ia32_psrlv32hi">,
1689               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
1690                         [IntrNoMem]>;
1691
1692   def int_x86_avx512_psrav_w_128 : GCCBuiltin<"__builtin_ia32_psrav8hi">,
1693               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
1694                         [IntrNoMem]>;
1695   def int_x86_avx512_psrav_w_256 : GCCBuiltin<"__builtin_ia32_psrav16hi">,
1696               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty],
1697                         [IntrNoMem]>;
1698   def int_x86_avx512_psrav_w_512 : GCCBuiltin<"__builtin_ia32_psrav32hi">,
1699               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
1700                         [IntrNoMem]>;
1701 }
1702
1703 // Gather ops
1704 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1705   // NOTE: These can't be ArgMemOnly because you can put the address completely
1706   // in the index register.
1707   def int_x86_avx2_gather_d_pd : GCCBuiltin<"__builtin_ia32_gatherd_pd">,
1708       Intrinsic<[llvm_v2f64_ty],
1709         [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty],
1710         [IntrReadMem, ImmArg<4>]>;
1711   def int_x86_avx2_gather_d_pd_256 : GCCBuiltin<"__builtin_ia32_gatherd_pd256">,
1712       Intrinsic<[llvm_v4f64_ty],
1713         [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty],
1714         [IntrReadMem, ImmArg<4>]>;
1715   def int_x86_avx2_gather_q_pd : GCCBuiltin<"__builtin_ia32_gatherq_pd">,
1716       Intrinsic<[llvm_v2f64_ty],
1717         [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty],
1718         [IntrReadMem, ImmArg<4>]>;
1719   def int_x86_avx2_gather_q_pd_256 : GCCBuiltin<"__builtin_ia32_gatherq_pd256">,
1720       Intrinsic<[llvm_v4f64_ty],
1721         [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty],
1722         [IntrReadMem, ImmArg<4>]>;
1723   def int_x86_avx2_gather_d_ps : GCCBuiltin<"__builtin_ia32_gatherd_ps">,
1724       Intrinsic<[llvm_v4f32_ty],
1725         [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty],
1726         [IntrReadMem, ImmArg<4>]>;
1727   def int_x86_avx2_gather_d_ps_256 : GCCBuiltin<"__builtin_ia32_gatherd_ps256">,
1728       Intrinsic<[llvm_v8f32_ty],
1729         [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty],
1730         [IntrReadMem, ImmArg<4>]>;
1731   def int_x86_avx2_gather_q_ps : GCCBuiltin<"__builtin_ia32_gatherq_ps">,
1732       Intrinsic<[llvm_v4f32_ty],
1733         [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty],
1734         [IntrReadMem, ImmArg<4>]>;
1735   def int_x86_avx2_gather_q_ps_256 : GCCBuiltin<"__builtin_ia32_gatherq_ps256">,
1736       Intrinsic<[llvm_v4f32_ty],
1737         [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty],
1738         [IntrReadMem, ImmArg<4>]>;
1739
1740   def int_x86_avx2_gather_d_q : GCCBuiltin<"__builtin_ia32_gatherd_q">,
1741       Intrinsic<[llvm_v2i64_ty],
1742         [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i8_ty],
1743         [IntrReadMem, ImmArg<4>]>;
1744   def int_x86_avx2_gather_d_q_256 : GCCBuiltin<"__builtin_ia32_gatherd_q256">,
1745       Intrinsic<[llvm_v4i64_ty],
1746         [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i8_ty],
1747         [IntrReadMem, ImmArg<4>]>;
1748   def int_x86_avx2_gather_q_q : GCCBuiltin<"__builtin_ia32_gatherq_q">,
1749       Intrinsic<[llvm_v2i64_ty],
1750         [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty],
1751         [IntrReadMem, ImmArg<4>]>;
1752   def int_x86_avx2_gather_q_q_256 : GCCBuiltin<"__builtin_ia32_gatherq_q256">,
1753       Intrinsic<[llvm_v4i64_ty],
1754         [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty],
1755         [IntrReadMem, ImmArg<4>]>;
1756   def int_x86_avx2_gather_d_d : GCCBuiltin<"__builtin_ia32_gatherd_d">,
1757       Intrinsic<[llvm_v4i32_ty],
1758         [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty],
1759         [IntrReadMem, ImmArg<4>]>;
1760   def int_x86_avx2_gather_d_d_256 : GCCBuiltin<"__builtin_ia32_gatherd_d256">,
1761       Intrinsic<[llvm_v8i32_ty],
1762         [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty],
1763         [IntrReadMem, ImmArg<4>]>;
1764   def int_x86_avx2_gather_q_d : GCCBuiltin<"__builtin_ia32_gatherq_d">,
1765       Intrinsic<[llvm_v4i32_ty],
1766         [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
1767         [IntrReadMem, ImmArg<4>]>;
1768   def int_x86_avx2_gather_q_d_256 : GCCBuiltin<"__builtin_ia32_gatherq_d256">,
1769       Intrinsic<[llvm_v4i32_ty],
1770         [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
1771         [IntrReadMem, ImmArg<4>]>;
1772 }
1773
1774 // Misc.
1775 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1776   def int_x86_avx2_pmovmskb : GCCBuiltin<"__builtin_ia32_pmovmskb256">,
1777               Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty], [IntrNoMem]>;
1778   def int_x86_avx2_pshuf_b : GCCBuiltin<"__builtin_ia32_pshufb256">,
1779               Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
1780                          llvm_v32i8_ty], [IntrNoMem]>;
1781   def int_x86_avx2_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw256">,
1782               Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
1783                          llvm_i8_ty], [IntrNoMem, Commutative, ImmArg<2>]>;
1784 }
1785
1786 //===----------------------------------------------------------------------===//
1787 // FMA3 and FMA4
1788
1789 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1790   def int_x86_avx512_vfmadd_pd_512 :
1791           Intrinsic<[llvm_v8f64_ty],
1792           [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty],
1793           [IntrNoMem, ImmArg<3>]>;
1794
1795   def int_x86_avx512_vfmadd_ps_512 :
1796           Intrinsic<[llvm_v16f32_ty],
1797           [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty],
1798           [IntrNoMem, ImmArg<3>]>;
1799
1800   // TODO: Can we use 2 vfmadds+shufflevector?
1801   def int_x86_avx512_vfmaddsub_pd_512 :
1802           Intrinsic<[llvm_v8f64_ty],
1803           [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty],
1804           [IntrNoMem, ImmArg<3>]>;
1805
1806   def int_x86_avx512_vfmaddsub_ps_512 :
1807           Intrinsic<[llvm_v16f32_ty],
1808           [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty],
1809           [IntrNoMem, ImmArg<3>]>;
1810
1811   def int_x86_avx512_vfmadd_f64 :
1812           Intrinsic<[llvm_double_ty],
1813                     [llvm_double_ty, llvm_double_ty, llvm_double_ty, llvm_i32_ty],
1814                     [IntrNoMem, ImmArg<3>]>;
1815   def int_x86_avx512_vfmadd_f32 :
1816           Intrinsic<[llvm_float_ty],
1817                     [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_i32_ty],
1818                     [IntrNoMem, ImmArg<3>]>;
1819
1820   def int_x86_avx512_vpmadd52h_uq_128 :
1821               GCCBuiltin<"__builtin_ia32_vpmadd52huq128">,
1822               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
1823                          llvm_v2i64_ty], [IntrNoMem]>;
1824   def int_x86_avx512_vpmadd52l_uq_128 :
1825               GCCBuiltin<"__builtin_ia32_vpmadd52luq128">,
1826               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
1827                          llvm_v2i64_ty], [IntrNoMem]>;
1828   def int_x86_avx512_vpmadd52h_uq_256 :
1829               GCCBuiltin<"__builtin_ia32_vpmadd52huq256">,
1830               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
1831                          llvm_v4i64_ty], [IntrNoMem]>;
1832   def int_x86_avx512_vpmadd52l_uq_256 :
1833               GCCBuiltin<"__builtin_ia32_vpmadd52luq256">,
1834               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
1835                          llvm_v4i64_ty], [IntrNoMem]>;
1836   def int_x86_avx512_vpmadd52h_uq_512 :
1837               GCCBuiltin<"__builtin_ia32_vpmadd52huq512">,
1838               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
1839                          llvm_v8i64_ty], [IntrNoMem]>;
1840   def int_x86_avx512_vpmadd52l_uq_512 :
1841               GCCBuiltin<"__builtin_ia32_vpmadd52luq512">,
1842               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
1843                          llvm_v8i64_ty], [IntrNoMem]>;
1844 }
1845
1846 // VNNI
1847 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1848   def int_x86_avx512_vpdpbusd_128 :
1849               GCCBuiltin<"__builtin_ia32_vpdpbusd128">,
1850               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
1851                          llvm_v4i32_ty], [IntrNoMem]>;
1852   def int_x86_avx512_vpdpbusd_256 :
1853               GCCBuiltin<"__builtin_ia32_vpdpbusd256">,
1854               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
1855                          llvm_v8i32_ty], [IntrNoMem]>;
1856   def int_x86_avx512_vpdpbusd_512 :
1857               GCCBuiltin<"__builtin_ia32_vpdpbusd512">,
1858               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
1859                          llvm_v16i32_ty], [IntrNoMem]>;
1860
1861   def int_x86_avx512_vpdpbusds_128 :
1862               GCCBuiltin<"__builtin_ia32_vpdpbusds128">,
1863               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
1864                          llvm_v4i32_ty], [IntrNoMem]>;
1865   def int_x86_avx512_vpdpbusds_256 :
1866               GCCBuiltin<"__builtin_ia32_vpdpbusds256">,
1867               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
1868                          llvm_v8i32_ty], [IntrNoMem]>;
1869   def int_x86_avx512_vpdpbusds_512 :
1870               GCCBuiltin<"__builtin_ia32_vpdpbusds512">,
1871               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
1872                          llvm_v16i32_ty], [IntrNoMem]>;
1873
1874   def int_x86_avx512_vpdpwssd_128 :
1875               GCCBuiltin<"__builtin_ia32_vpdpwssd128">,
1876               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
1877                          llvm_v4i32_ty], [IntrNoMem]>;
1878   def int_x86_avx512_vpdpwssd_256 :
1879               GCCBuiltin<"__builtin_ia32_vpdpwssd256">,
1880               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
1881                          llvm_v8i32_ty], [IntrNoMem]>;
1882   def int_x86_avx512_vpdpwssd_512 :
1883               GCCBuiltin<"__builtin_ia32_vpdpwssd512">,
1884               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
1885                          llvm_v16i32_ty], [IntrNoMem]>;
1886
1887   def int_x86_avx512_vpdpwssds_128 :
1888               GCCBuiltin<"__builtin_ia32_vpdpwssds128">,
1889               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
1890                          llvm_v4i32_ty], [IntrNoMem]>;
1891   def int_x86_avx512_vpdpwssds_256 :
1892               GCCBuiltin<"__builtin_ia32_vpdpwssds256">,
1893               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
1894                          llvm_v8i32_ty], [IntrNoMem]>;
1895   def int_x86_avx512_vpdpwssds_512 :
1896               GCCBuiltin<"__builtin_ia32_vpdpwssds512">,
1897               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
1898                          llvm_v16i32_ty], [IntrNoMem]>;
1899 }
1900
1901 //===----------------------------------------------------------------------===//
1902 // XOP
1903
1904 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
1905   def int_x86_xop_vpermil2pd : GCCBuiltin<"__builtin_ia32_vpermil2pd">,
1906               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
1907                                           llvm_v2i64_ty, llvm_i8_ty],
1908                         [IntrNoMem, ImmArg<3>]>;
1909
1910   def int_x86_xop_vpermil2pd_256 :
1911               GCCBuiltin<"__builtin_ia32_vpermil2pd256">,
1912               Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
1913                                           llvm_v4i64_ty, llvm_i8_ty],
1914                         [IntrNoMem, ImmArg<3>]>;
1915
1916   def int_x86_xop_vpermil2ps : GCCBuiltin<"__builtin_ia32_vpermil2ps">,
1917               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
1918                                           llvm_v4i32_ty, llvm_i8_ty],
1919                         [IntrNoMem, ImmArg<3>]>;
1920   def int_x86_xop_vpermil2ps_256 :
1921               GCCBuiltin<"__builtin_ia32_vpermil2ps256">,
1922               Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
1923                                           llvm_v8i32_ty, llvm_i8_ty],
1924                         [IntrNoMem, ImmArg<3>]>;
1925
1926   def int_x86_xop_vfrcz_pd : GCCBuiltin<"__builtin_ia32_vfrczpd">,
1927               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
1928   def int_x86_xop_vfrcz_ps : GCCBuiltin<"__builtin_ia32_vfrczps">,
1929               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
1930   def int_x86_xop_vfrcz_sd : GCCBuiltin<"__builtin_ia32_vfrczsd">,
1931               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
1932   def int_x86_xop_vfrcz_ss : GCCBuiltin<"__builtin_ia32_vfrczss">,
1933               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
1934   def int_x86_xop_vfrcz_pd_256 : GCCBuiltin<"__builtin_ia32_vfrczpd256">,
1935               Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty], [IntrNoMem]>;
1936   def int_x86_xop_vfrcz_ps_256 : GCCBuiltin<"__builtin_ia32_vfrczps256">,
1937               Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
1938
1939   def int_x86_xop_vphaddbd :
1940               GCCBuiltin<"__builtin_ia32_vphaddbd">,
1941               Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1942   def int_x86_xop_vphaddbq :
1943               GCCBuiltin<"__builtin_ia32_vphaddbq">,
1944               Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1945   def int_x86_xop_vphaddbw :
1946               GCCBuiltin<"__builtin_ia32_vphaddbw">,
1947               Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1948   def int_x86_xop_vphadddq :
1949               GCCBuiltin<"__builtin_ia32_vphadddq">,
1950               Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>;
1951   def int_x86_xop_vphaddubd :
1952               GCCBuiltin<"__builtin_ia32_vphaddubd">,
1953               Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1954   def int_x86_xop_vphaddubq :
1955               GCCBuiltin<"__builtin_ia32_vphaddubq">,
1956               Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1957   def int_x86_xop_vphaddubw :
1958               GCCBuiltin<"__builtin_ia32_vphaddubw">,
1959               Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1960   def int_x86_xop_vphaddudq :
1961               GCCBuiltin<"__builtin_ia32_vphaddudq">,
1962               Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>;
1963   def int_x86_xop_vphadduwd :
1964               GCCBuiltin<"__builtin_ia32_vphadduwd">,
1965               Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
1966   def int_x86_xop_vphadduwq :
1967               GCCBuiltin<"__builtin_ia32_vphadduwq">,
1968               Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], [IntrNoMem]>;
1969   def int_x86_xop_vphaddwd :
1970               GCCBuiltin<"__builtin_ia32_vphaddwd">,
1971               Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
1972   def int_x86_xop_vphaddwq :
1973               GCCBuiltin<"__builtin_ia32_vphaddwq">,
1974               Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], [IntrNoMem]>;
1975   def int_x86_xop_vphsubbw :
1976               GCCBuiltin<"__builtin_ia32_vphsubbw">,
1977               Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>;
1978   def int_x86_xop_vphsubdq :
1979               GCCBuiltin<"__builtin_ia32_vphsubdq">,
1980               Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>;
1981   def int_x86_xop_vphsubwd :
1982               GCCBuiltin<"__builtin_ia32_vphsubwd">,
1983               Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
1984   def int_x86_xop_vpmacsdd :
1985               GCCBuiltin<"__builtin_ia32_vpmacsdd">,
1986               Intrinsic<[llvm_v4i32_ty],
1987                         [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
1988                         [IntrNoMem, Commutative]>;
1989   def int_x86_xop_vpmacsdqh :
1990               GCCBuiltin<"__builtin_ia32_vpmacsdqh">,
1991               Intrinsic<[llvm_v2i64_ty],
1992                         [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty],
1993                         [IntrNoMem, Commutative]>;
1994   def int_x86_xop_vpmacsdql :
1995               GCCBuiltin<"__builtin_ia32_vpmacsdql">,
1996               Intrinsic<[llvm_v2i64_ty],
1997                         [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty],
1998                         [IntrNoMem, Commutative]>;
1999   def int_x86_xop_vpmacssdd :
2000               GCCBuiltin<"__builtin_ia32_vpmacssdd">,
2001               Intrinsic<[llvm_v4i32_ty],
2002                         [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
2003                         [IntrNoMem, Commutative]>;
2004   def int_x86_xop_vpmacssdqh :
2005               GCCBuiltin<"__builtin_ia32_vpmacssdqh">,
2006               Intrinsic<[llvm_v2i64_ty],
2007                         [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty],
2008                         [IntrNoMem, Commutative]>;
2009   def int_x86_xop_vpmacssdql :
2010               GCCBuiltin<"__builtin_ia32_vpmacssdql">,
2011               Intrinsic<[llvm_v2i64_ty],
2012                         [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v2i64_ty],
2013                         [IntrNoMem, Commutative]>;
2014   def int_x86_xop_vpmacsswd :
2015               GCCBuiltin<"__builtin_ia32_vpmacsswd">,
2016               Intrinsic<[llvm_v4i32_ty],
2017                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty],
2018                         [IntrNoMem, Commutative]>;
2019   def int_x86_xop_vpmacssww :
2020               GCCBuiltin<"__builtin_ia32_vpmacssww">,
2021               Intrinsic<[llvm_v8i16_ty],
2022                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty],
2023                         [IntrNoMem, Commutative]>;
2024   def int_x86_xop_vpmacswd :
2025               GCCBuiltin<"__builtin_ia32_vpmacswd">,
2026               Intrinsic<[llvm_v4i32_ty],
2027                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty],
2028                         [IntrNoMem, Commutative]>;
2029   def int_x86_xop_vpmacsww :
2030               GCCBuiltin<"__builtin_ia32_vpmacsww">,
2031               Intrinsic<[llvm_v8i16_ty],
2032                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty],
2033                         [IntrNoMem, Commutative]>;
2034   def int_x86_xop_vpmadcsswd :
2035               GCCBuiltin<"__builtin_ia32_vpmadcsswd">,
2036               Intrinsic<[llvm_v4i32_ty],
2037                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty],
2038                         [IntrNoMem, Commutative]>;
2039   def int_x86_xop_vpmadcswd :
2040               GCCBuiltin<"__builtin_ia32_vpmadcswd">,
2041               Intrinsic<[llvm_v4i32_ty],
2042                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty],
2043                         [IntrNoMem, Commutative]>;
2044   def int_x86_xop_vpperm :
2045               GCCBuiltin<"__builtin_ia32_vpperm">,
2046               Intrinsic<[llvm_v16i8_ty],
2047                         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
2048                         [IntrNoMem]>;
2049   def int_x86_xop_vpshab :
2050               GCCBuiltin<"__builtin_ia32_vpshab">,
2051               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty],
2052                         [IntrNoMem]>;
2053   def int_x86_xop_vpshad :
2054               GCCBuiltin<"__builtin_ia32_vpshad">,
2055               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
2056                         [IntrNoMem]>;
2057   def int_x86_xop_vpshaq :
2058               GCCBuiltin<"__builtin_ia32_vpshaq">,
2059               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
2060                         [IntrNoMem]>;
2061   def int_x86_xop_vpshaw :
2062               GCCBuiltin<"__builtin_ia32_vpshaw">,
2063               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
2064                         [IntrNoMem]>;
2065   def int_x86_xop_vpshlb :
2066               GCCBuiltin<"__builtin_ia32_vpshlb">,
2067               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty],
2068                         [IntrNoMem]>;
2069   def int_x86_xop_vpshld :
2070               GCCBuiltin<"__builtin_ia32_vpshld">,
2071               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
2072                         [IntrNoMem]>;
2073   def int_x86_xop_vpshlq :
2074               GCCBuiltin<"__builtin_ia32_vpshlq">,
2075               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
2076                         [IntrNoMem]>;
2077   def int_x86_xop_vpshlw :
2078               GCCBuiltin<"__builtin_ia32_vpshlw">,
2079               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
2080                         [IntrNoMem]>;
2081 }
2082
2083 //===----------------------------------------------------------------------===//
2084 // LWP
2085 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2086   def int_x86_llwpcb :
2087               GCCBuiltin<"__builtin_ia32_llwpcb">,
2088               Intrinsic<[], [llvm_ptr_ty], []>;
2089   def int_x86_slwpcb :
2090               GCCBuiltin<"__builtin_ia32_slwpcb">,
2091               Intrinsic<[llvm_ptr_ty], [], []>;
2092   def int_x86_lwpins32 :
2093               GCCBuiltin<"__builtin_ia32_lwpins32">,
2094               Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
2095                         [ImmArg<2>]>;
2096   def int_x86_lwpins64 :
2097               GCCBuiltin<"__builtin_ia32_lwpins64">,
2098               Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty],
2099                         [ImmArg<2>]>;
2100   def int_x86_lwpval32 :
2101               GCCBuiltin<"__builtin_ia32_lwpval32">,
2102               Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
2103                         [ImmArg<2>]>;
2104   def int_x86_lwpval64 :
2105               GCCBuiltin<"__builtin_ia32_lwpval64">,
2106               Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty],
2107                         [ImmArg<2>]>;
2108 }
2109
2110 //===----------------------------------------------------------------------===//
2111 // MMX
2112
2113 // Empty MMX state op.
2114 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2115   def int_x86_mmx_emms  : GCCBuiltin<"__builtin_ia32_emms">,
2116               Intrinsic<[], [], []>;
2117   def int_x86_mmx_femms : GCCBuiltin<"__builtin_ia32_femms">,
2118               Intrinsic<[], [], []>;
2119 }
2120
2121 // Integer arithmetic ops.
2122 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2123   // Addition
2124   def int_x86_mmx_padd_b : GCCBuiltin<"__builtin_ia32_paddb">,
2125               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2126                         [IntrNoMem, Commutative]>;
2127   def int_x86_mmx_padd_w : GCCBuiltin<"__builtin_ia32_paddw">,
2128               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2129                         [IntrNoMem, Commutative]>;
2130   def int_x86_mmx_padd_d : GCCBuiltin<"__builtin_ia32_paddd">,
2131               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2132                         [IntrNoMem, Commutative]>;
2133   def int_x86_mmx_padd_q : GCCBuiltin<"__builtin_ia32_paddq">,
2134               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2135                         [IntrNoMem, Commutative]>;
2136
2137   def int_x86_mmx_padds_b : GCCBuiltin<"__builtin_ia32_paddsb">,
2138               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2139                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2140   def int_x86_mmx_padds_w : GCCBuiltin<"__builtin_ia32_paddsw">,
2141               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2142                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2143
2144   def int_x86_mmx_paddus_b : GCCBuiltin<"__builtin_ia32_paddusb">,
2145               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2146                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2147   def int_x86_mmx_paddus_w : GCCBuiltin<"__builtin_ia32_paddusw">,
2148               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2149                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2150
2151   // Subtraction
2152   def int_x86_mmx_psub_b : GCCBuiltin<"__builtin_ia32_psubb">,
2153               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2154                         [IntrNoMem]>;
2155   def int_x86_mmx_psub_w : GCCBuiltin<"__builtin_ia32_psubw">,
2156               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2157                         [IntrNoMem]>;
2158   def int_x86_mmx_psub_d : GCCBuiltin<"__builtin_ia32_psubd">,
2159               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2160                         [IntrNoMem]>;
2161   def int_x86_mmx_psub_q : GCCBuiltin<"__builtin_ia32_psubq">,
2162               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2163                         [IntrNoMem]>;
2164
2165   def int_x86_mmx_psubs_b : GCCBuiltin<"__builtin_ia32_psubsb">,
2166               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2167                          llvm_x86mmx_ty], [IntrNoMem]>;
2168   def int_x86_mmx_psubs_w : GCCBuiltin<"__builtin_ia32_psubsw">,
2169               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2170                          llvm_x86mmx_ty], [IntrNoMem]>;
2171
2172   def int_x86_mmx_psubus_b : GCCBuiltin<"__builtin_ia32_psubusb">,
2173               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2174                          llvm_x86mmx_ty], [IntrNoMem]>;
2175   def int_x86_mmx_psubus_w : GCCBuiltin<"__builtin_ia32_psubusw">,
2176               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2177                          llvm_x86mmx_ty], [IntrNoMem]>;
2178
2179   // Multiplication
2180   def int_x86_mmx_pmulh_w : GCCBuiltin<"__builtin_ia32_pmulhw">,
2181               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2182                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2183   def int_x86_mmx_pmull_w : GCCBuiltin<"__builtin_ia32_pmullw">,
2184               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2185                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2186   def int_x86_mmx_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw">,
2187               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2188                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2189   def int_x86_mmx_pmulu_dq : GCCBuiltin<"__builtin_ia32_pmuludq">,
2190               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2191                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2192   def int_x86_mmx_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd">,
2193               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2194                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2195
2196   // Bitwise operations
2197   def int_x86_mmx_pand : GCCBuiltin<"__builtin_ia32_pand">,
2198               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2199                         [IntrNoMem, Commutative]>;
2200   def int_x86_mmx_pandn : GCCBuiltin<"__builtin_ia32_pandn">,
2201               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2202                         [IntrNoMem]>;
2203   def int_x86_mmx_por : GCCBuiltin<"__builtin_ia32_por">,
2204               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2205                         [IntrNoMem, Commutative]>;
2206   def int_x86_mmx_pxor : GCCBuiltin<"__builtin_ia32_pxor">,
2207               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2208                         [IntrNoMem, Commutative]>;
2209
2210   // Averages
2211   def int_x86_mmx_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb">,
2212               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2213                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2214   def int_x86_mmx_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw">,
2215               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2216                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2217
2218   // Maximum
2219   def int_x86_mmx_pmaxu_b : GCCBuiltin<"__builtin_ia32_pmaxub">,
2220               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2221                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2222   def int_x86_mmx_pmaxs_w : GCCBuiltin<"__builtin_ia32_pmaxsw">,
2223               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2224                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2225
2226   // Minimum
2227   def int_x86_mmx_pminu_b : GCCBuiltin<"__builtin_ia32_pminub">,
2228               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2229                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2230   def int_x86_mmx_pmins_w : GCCBuiltin<"__builtin_ia32_pminsw">,
2231               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2232                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2233
2234   // Packed sum of absolute differences
2235   def int_x86_mmx_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw">,
2236               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2237                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2238 }
2239
2240 // Integer shift ops.
2241 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2242   // Shift left logical
2243   def int_x86_mmx_psll_w : GCCBuiltin<"__builtin_ia32_psllw">,
2244               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2245                          llvm_x86mmx_ty], [IntrNoMem]>;
2246   def int_x86_mmx_psll_d : GCCBuiltin<"__builtin_ia32_pslld">,
2247               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2248                          llvm_x86mmx_ty], [IntrNoMem]>;
2249   def int_x86_mmx_psll_q : GCCBuiltin<"__builtin_ia32_psllq">,
2250               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2251                          llvm_x86mmx_ty], [IntrNoMem]>;
2252
2253   def int_x86_mmx_psrl_w : GCCBuiltin<"__builtin_ia32_psrlw">,
2254               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2255                          llvm_x86mmx_ty], [IntrNoMem]>;
2256   def int_x86_mmx_psrl_d : GCCBuiltin<"__builtin_ia32_psrld">,
2257               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2258                          llvm_x86mmx_ty], [IntrNoMem]>;
2259   def int_x86_mmx_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq">,
2260               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2261                          llvm_x86mmx_ty], [IntrNoMem]>;
2262
2263   def int_x86_mmx_psra_w : GCCBuiltin<"__builtin_ia32_psraw">,
2264               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2265                          llvm_x86mmx_ty], [IntrNoMem]>;
2266   def int_x86_mmx_psra_d : GCCBuiltin<"__builtin_ia32_psrad">,
2267               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2268                          llvm_x86mmx_ty], [IntrNoMem]>;
2269
2270   // Oddly these don't require an immediate due to a gcc compatibility issue.
2271   def int_x86_mmx_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi">,
2272               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2273                          llvm_i32_ty], [IntrNoMem]>;
2274   def int_x86_mmx_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi">,
2275               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2276                          llvm_i32_ty], [IntrNoMem]>;
2277   def int_x86_mmx_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi">,
2278               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2279                          llvm_i32_ty], [IntrNoMem]>;
2280
2281   def int_x86_mmx_psrli_w : GCCBuiltin<"__builtin_ia32_psrlwi">,
2282               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2283                          llvm_i32_ty], [IntrNoMem]>;
2284   def int_x86_mmx_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi">,
2285               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2286                          llvm_i32_ty], [IntrNoMem]>;
2287   def int_x86_mmx_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi">,
2288               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2289                          llvm_i32_ty], [IntrNoMem]>;
2290
2291   def int_x86_mmx_psrai_w : GCCBuiltin<"__builtin_ia32_psrawi">,
2292               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2293                          llvm_i32_ty], [IntrNoMem]>;
2294   def int_x86_mmx_psrai_d : GCCBuiltin<"__builtin_ia32_psradi">,
2295               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2296                          llvm_i32_ty], [IntrNoMem]>;
2297 }
2298 // Permute
2299 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2300   def int_x86_avx512_permvar_df_256 : GCCBuiltin<"__builtin_ia32_permvardf256">,
2301               Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
2302                         llvm_v4i64_ty],  [IntrNoMem]>;
2303   def int_x86_avx512_permvar_df_512 : GCCBuiltin<"__builtin_ia32_permvardf512">,
2304               Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty,
2305                         llvm_v8i64_ty],  [IntrNoMem]>;
2306   def int_x86_avx512_permvar_di_256 : GCCBuiltin<"__builtin_ia32_permvardi256">,
2307               Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
2308                         llvm_v4i64_ty],  [IntrNoMem]>;
2309   def int_x86_avx512_permvar_di_512 : GCCBuiltin<"__builtin_ia32_permvardi512">,
2310               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
2311                         llvm_v8i64_ty],  [IntrNoMem]>;
2312   def int_x86_avx512_permvar_hi_128 : GCCBuiltin<"__builtin_ia32_permvarhi128">,
2313               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
2314                         llvm_v8i16_ty],  [IntrNoMem]>;
2315   def int_x86_avx512_permvar_hi_256 : GCCBuiltin<"__builtin_ia32_permvarhi256">,
2316               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
2317                         llvm_v16i16_ty],  [IntrNoMem]>;
2318   def int_x86_avx512_permvar_hi_512 : GCCBuiltin<"__builtin_ia32_permvarhi512">,
2319               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
2320                         llvm_v32i16_ty],  [IntrNoMem]>;
2321   def int_x86_avx512_permvar_qi_128 : GCCBuiltin<"__builtin_ia32_permvarqi128">,
2322               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
2323                         llvm_v16i8_ty],  [IntrNoMem]>;
2324   def int_x86_avx512_permvar_qi_256 : GCCBuiltin<"__builtin_ia32_permvarqi256">,
2325               Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
2326                         llvm_v32i8_ty],  [IntrNoMem]>;
2327   def int_x86_avx512_permvar_qi_512 : GCCBuiltin<"__builtin_ia32_permvarqi512">,
2328               Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty,
2329                         llvm_v64i8_ty],  [IntrNoMem]>;
2330   def int_x86_avx512_permvar_sf_512 : GCCBuiltin<"__builtin_ia32_permvarsf512">,
2331               Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty,
2332                         llvm_v16i32_ty],  [IntrNoMem]>;
2333   def int_x86_avx512_permvar_si_512 : GCCBuiltin<"__builtin_ia32_permvarsi512">,
2334               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
2335                         llvm_v16i32_ty],  [IntrNoMem]>;
2336 }
2337 // Pack ops.
2338 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2339   def int_x86_mmx_packsswb : GCCBuiltin<"__builtin_ia32_packsswb">,
2340               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2341                          llvm_x86mmx_ty], [IntrNoMem]>;
2342   def int_x86_mmx_packssdw : GCCBuiltin<"__builtin_ia32_packssdw">,
2343               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2344                          llvm_x86mmx_ty], [IntrNoMem]>;
2345   def int_x86_mmx_packuswb : GCCBuiltin<"__builtin_ia32_packuswb">,
2346               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2347                          llvm_x86mmx_ty], [IntrNoMem]>;
2348 }
2349
2350 // Unpacking ops.
2351 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2352   def int_x86_mmx_punpckhbw : GCCBuiltin<"__builtin_ia32_punpckhbw">,
2353               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2354                         [IntrNoMem]>;
2355   def int_x86_mmx_punpckhwd : GCCBuiltin<"__builtin_ia32_punpckhwd">,
2356               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2357                         [IntrNoMem]>;
2358   def int_x86_mmx_punpckhdq : GCCBuiltin<"__builtin_ia32_punpckhdq">,
2359               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2360                         [IntrNoMem]>;
2361   def int_x86_mmx_punpcklbw : GCCBuiltin<"__builtin_ia32_punpcklbw">,
2362               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2363                         [IntrNoMem]>;
2364   def int_x86_mmx_punpcklwd : GCCBuiltin<"__builtin_ia32_punpcklwd">,
2365               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2366                         [IntrNoMem]>;
2367   def int_x86_mmx_punpckldq : GCCBuiltin<"__builtin_ia32_punpckldq">,
2368               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty],
2369                         [IntrNoMem]>;
2370 }
2371
2372 // Integer comparison ops
2373 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2374   def int_x86_mmx_pcmpeq_b : GCCBuiltin<"__builtin_ia32_pcmpeqb">,
2375               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2376                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2377   def int_x86_mmx_pcmpeq_w : GCCBuiltin<"__builtin_ia32_pcmpeqw">,
2378               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2379                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2380   def int_x86_mmx_pcmpeq_d : GCCBuiltin<"__builtin_ia32_pcmpeqd">,
2381               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2382                          llvm_x86mmx_ty], [IntrNoMem, Commutative]>;
2383
2384   def int_x86_mmx_pcmpgt_b : GCCBuiltin<"__builtin_ia32_pcmpgtb">,
2385               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2386                          llvm_x86mmx_ty], [IntrNoMem]>;
2387   def int_x86_mmx_pcmpgt_w : GCCBuiltin<"__builtin_ia32_pcmpgtw">,
2388               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2389                          llvm_x86mmx_ty], [IntrNoMem]>;
2390   def int_x86_mmx_pcmpgt_d : GCCBuiltin<"__builtin_ia32_pcmpgtd">,
2391               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2392                          llvm_x86mmx_ty], [IntrNoMem]>;
2393 }
2394
2395 // Misc.
2396 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2397   def int_x86_mmx_maskmovq : GCCBuiltin<"__builtin_ia32_maskmovq">,
2398               Intrinsic<[], [llvm_x86mmx_ty, llvm_x86mmx_ty, llvm_ptr_ty], []>;
2399
2400   def int_x86_mmx_pmovmskb : GCCBuiltin<"__builtin_ia32_pmovmskb">,
2401               Intrinsic<[llvm_i32_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
2402
2403   def int_x86_mmx_movnt_dq : GCCBuiltin<"__builtin_ia32_movntq">,
2404               Intrinsic<[], [llvm_ptrx86mmx_ty, llvm_x86mmx_ty], []>;
2405
2406   def int_x86_mmx_palignr_b : GCCBuiltin<"__builtin_ia32_palignr">,
2407               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2408                         llvm_x86mmx_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>;
2409
2410   def int_x86_mmx_pextr_w : GCCBuiltin<"__builtin_ia32_vec_ext_v4hi">,
2411               Intrinsic<[llvm_i32_ty], [llvm_x86mmx_ty, llvm_i32_ty],
2412                         [IntrNoMem, ImmArg<1>]>;
2413
2414   def int_x86_mmx_pinsr_w : GCCBuiltin<"__builtin_ia32_vec_set_v4hi">,
2415               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty,
2416                         llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2417 }
2418
2419 //===----------------------------------------------------------------------===//
2420 // BMI
2421
2422 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2423   def int_x86_bmi_bextr_32 : GCCBuiltin<"__builtin_ia32_bextr_u32">,
2424               Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
2425   def int_x86_bmi_bextr_64 : GCCBuiltin<"__builtin_ia32_bextr_u64">,
2426               Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
2427   def int_x86_bmi_bzhi_32 : GCCBuiltin<"__builtin_ia32_bzhi_si">,
2428               Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
2429   def int_x86_bmi_bzhi_64 : GCCBuiltin<"__builtin_ia32_bzhi_di">,
2430               Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
2431   def int_x86_bmi_pdep_32 : GCCBuiltin<"__builtin_ia32_pdep_si">,
2432               Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
2433   def int_x86_bmi_pdep_64 : GCCBuiltin<"__builtin_ia32_pdep_di">,
2434               Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
2435   def int_x86_bmi_pext_32 : GCCBuiltin<"__builtin_ia32_pext_si">,
2436               Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
2437   def int_x86_bmi_pext_64 : GCCBuiltin<"__builtin_ia32_pext_di">,
2438               Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
2439 }
2440
2441 //===----------------------------------------------------------------------===//
2442 // FS/GS Base
2443
2444 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2445   def int_x86_rdfsbase_32 : GCCBuiltin<"__builtin_ia32_rdfsbase32">,
2446               Intrinsic<[llvm_i32_ty], []>;
2447   def int_x86_rdgsbase_32 : GCCBuiltin<"__builtin_ia32_rdgsbase32">,
2448               Intrinsic<[llvm_i32_ty], []>;
2449   def int_x86_rdfsbase_64 : GCCBuiltin<"__builtin_ia32_rdfsbase64">,
2450               Intrinsic<[llvm_i64_ty], []>;
2451   def int_x86_rdgsbase_64 : GCCBuiltin<"__builtin_ia32_rdgsbase64">,
2452               Intrinsic<[llvm_i64_ty], []>;
2453   def int_x86_wrfsbase_32 : GCCBuiltin<"__builtin_ia32_wrfsbase32">,
2454               Intrinsic<[], [llvm_i32_ty]>;
2455   def int_x86_wrgsbase_32 : GCCBuiltin<"__builtin_ia32_wrgsbase32">,
2456               Intrinsic<[], [llvm_i32_ty]>;
2457   def int_x86_wrfsbase_64 : GCCBuiltin<"__builtin_ia32_wrfsbase64">,
2458               Intrinsic<[], [llvm_i64_ty]>;
2459   def int_x86_wrgsbase_64 : GCCBuiltin<"__builtin_ia32_wrgsbase64">,
2460               Intrinsic<[], [llvm_i64_ty]>;
2461 }
2462
2463 //===----------------------------------------------------------------------===//
2464 // FXSR
2465 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2466   def int_x86_fxrstor : GCCBuiltin<"__builtin_ia32_fxrstor">,
2467               Intrinsic<[], [llvm_ptr_ty], []>;
2468   def int_x86_fxrstor64 : GCCBuiltin<"__builtin_ia32_fxrstor64">,
2469               Intrinsic<[], [llvm_ptr_ty], []>;
2470   def int_x86_fxsave : GCCBuiltin<"__builtin_ia32_fxsave">,
2471               Intrinsic<[], [llvm_ptr_ty], []>;
2472   def int_x86_fxsave64 : GCCBuiltin<"__builtin_ia32_fxsave64">,
2473               Intrinsic<[], [llvm_ptr_ty], []>;
2474 }
2475
2476 //===----------------------------------------------------------------------===//
2477 // XSAVE
2478 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2479   def int_x86_xsave :
2480               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2481   def int_x86_xsave64 :
2482               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2483   def int_x86_xrstor :
2484               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2485   def int_x86_xrstor64 :
2486               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2487   def int_x86_xsaveopt :
2488               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2489   def int_x86_xsaveopt64 :
2490               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2491   def int_x86_xrstors :
2492               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2493   def int_x86_xrstors64 :
2494               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2495   def int_x86_xsavec :
2496               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2497   def int_x86_xsavec64 :
2498               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2499   def int_x86_xsaves :
2500               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2501   def int_x86_xsaves64 :
2502               Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>;
2503   def int_x86_xgetbv :
2504               Intrinsic<[llvm_i64_ty], [llvm_i32_ty], []>;
2505   def int_x86_xsetbv :
2506               Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
2507 }
2508
2509 //===----------------------------------------------------------------------===//
2510 // CLFLUSHOPT and CLWB
2511 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2512   def int_x86_clflushopt : GCCBuiltin<"__builtin_ia32_clflushopt">,
2513               Intrinsic<[], [llvm_ptr_ty], []>;
2514
2515   def int_x86_clwb : GCCBuiltin<"__builtin_ia32_clwb">,
2516               Intrinsic<[], [llvm_ptr_ty], []>;
2517 }
2518
2519 //===----------------------------------------------------------------------===//
2520 // Support protection key
2521 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2522   def int_x86_rdpkru : GCCBuiltin <"__builtin_ia32_rdpkru">,
2523               Intrinsic<[llvm_i32_ty], [], []>;
2524   def int_x86_wrpkru : GCCBuiltin<"__builtin_ia32_wrpkru">,
2525               Intrinsic<[], [llvm_i32_ty], []>;
2526 }
2527 //===----------------------------------------------------------------------===//
2528 // Half float conversion
2529
2530 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2531   def int_x86_vcvtph2ps_128 : GCCBuiltin<"__builtin_ia32_vcvtph2ps">,
2532               Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
2533   def int_x86_vcvtph2ps_256 : GCCBuiltin<"__builtin_ia32_vcvtph2ps256">,
2534               Intrinsic<[llvm_v8f32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
2535   def int_x86_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph">,
2536               Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty],
2537                         [IntrNoMem, ImmArg<1>]>;
2538   def int_x86_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256">,
2539               Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty],
2540                         [IntrNoMem, ImmArg<1>]>;
2541   def int_x86_avx512_mask_vcvtph2ps_512 : GCCBuiltin<"__builtin_ia32_vcvtph2ps512_mask">,
2542               Intrinsic<[llvm_v16f32_ty], [llvm_v16i16_ty, llvm_v16f32_ty,
2543                                            llvm_i16_ty, llvm_i32_ty],
2544                         [IntrNoMem, ImmArg<3>]>;
2545   def int_x86_avx512_mask_vcvtph2ps_256 : GCCBuiltin<"__builtin_ia32_vcvtph2ps256_mask">,
2546               Intrinsic<[llvm_v8f32_ty], [llvm_v8i16_ty, llvm_v8f32_ty,
2547                                            llvm_i8_ty], [IntrNoMem]>;
2548   def int_x86_avx512_mask_vcvtph2ps_128 : GCCBuiltin<"__builtin_ia32_vcvtph2ps_mask">,
2549               Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty, llvm_v4f32_ty,
2550                                            llvm_i8_ty], [IntrNoMem]>;
2551   def int_x86_avx512_mask_vcvtps2ph_512 : GCCBuiltin<"__builtin_ia32_vcvtps2ph512_mask">,
2552               Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty, llvm_i32_ty,
2553                                            llvm_v16i16_ty, llvm_i16_ty],
2554                         [IntrNoMem, ImmArg<1>]>;
2555   def int_x86_avx512_mask_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256_mask">,
2556               Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty,
2557                                            llvm_v8i16_ty, llvm_i8_ty],
2558                         [IntrNoMem, ImmArg<1>]>;
2559   def int_x86_avx512_mask_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph_mask">,
2560               Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty,
2561                                            llvm_v8i16_ty, llvm_i8_ty],
2562                         [IntrNoMem, ImmArg<1>]>;
2563 }
2564
2565 //===----------------------------------------------------------------------===//
2566 // TBM
2567
2568 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2569   def int_x86_tbm_bextri_u32 : GCCBuiltin<"__builtin_ia32_bextri_u32">,
2570         Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
2571                   [IntrNoMem, ImmArg<1>]>;
2572   def int_x86_tbm_bextri_u64 : GCCBuiltin<"__builtin_ia32_bextri_u64">,
2573         Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty],
2574                   [IntrNoMem, ImmArg<1>]>;
2575 }
2576
2577 //===----------------------------------------------------------------------===//
2578 // RDRAND intrinsics - Return a random value and whether it is valid.
2579 // RDSEED intrinsics - Return a NIST SP800-90B & C compliant random value and
2580 // whether it is valid.
2581
2582 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2583   // These are declared side-effecting so they don't get eliminated by CSE or
2584   // LICM.
2585   def int_x86_rdrand_16 : Intrinsic<[llvm_i16_ty, llvm_i32_ty], [], []>;
2586   def int_x86_rdrand_32 : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [], []>;
2587   def int_x86_rdrand_64 : Intrinsic<[llvm_i64_ty, llvm_i32_ty], [], []>;
2588   def int_x86_rdseed_16 : Intrinsic<[llvm_i16_ty, llvm_i32_ty], [], []>;
2589   def int_x86_rdseed_32 : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [], []>;
2590   def int_x86_rdseed_64 : Intrinsic<[llvm_i64_ty, llvm_i32_ty], [], []>;
2591 }
2592
2593 //===----------------------------------------------------------------------===//
2594 // ADX
2595
2596 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2597   def int_x86_addcarry_32:
2598         Intrinsic<[llvm_i8_ty, llvm_i32_ty],
2599                   [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
2600   def int_x86_addcarry_64:
2601         Intrinsic<[llvm_i8_ty, llvm_i64_ty],
2602                   [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
2603   def int_x86_subborrow_32:
2604         Intrinsic<[llvm_i8_ty, llvm_i32_ty],
2605                   [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
2606   def int_x86_subborrow_64:
2607         Intrinsic<[llvm_i8_ty, llvm_i64_ty],
2608                   [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
2609 }
2610
2611 //===----------------------------------------------------------------------===//
2612 // RTM intrinsics. Transactional Memory support.
2613
2614 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2615   def int_x86_xbegin : GCCBuiltin<"__builtin_ia32_xbegin">,
2616               Intrinsic<[llvm_i32_ty], [], []>;
2617   def int_x86_xend : GCCBuiltin<"__builtin_ia32_xend">,
2618               Intrinsic<[], [], []>;
2619   def int_x86_xabort : GCCBuiltin<"__builtin_ia32_xabort">,
2620               Intrinsic<[], [llvm_i8_ty], [ImmArg<0>]>;
2621   def int_x86_xtest : GCCBuiltin<"__builtin_ia32_xtest">,
2622               Intrinsic<[llvm_i32_ty], [], []>;
2623 }
2624
2625 //===----------------------------------------------------------------------===//
2626 // AVX512
2627
2628 // Mask ops
2629 let TargetPrefix = "x86" in {
2630   def int_x86_avx512_kadd_b :
2631         Intrinsic<[llvm_v8i1_ty], [llvm_v8i1_ty, llvm_v8i1_ty], [IntrNoMem]>;
2632   def int_x86_avx512_kadd_w :
2633         Intrinsic<[llvm_v16i1_ty], [llvm_v16i1_ty, llvm_v16i1_ty], [IntrNoMem]>;
2634   def int_x86_avx512_kadd_d :
2635         Intrinsic<[llvm_v32i1_ty], [llvm_v32i1_ty, llvm_v32i1_ty], [IntrNoMem]>;
2636   def int_x86_avx512_kadd_q :
2637         Intrinsic<[llvm_v64i1_ty], [llvm_v64i1_ty, llvm_v64i1_ty], [IntrNoMem]>;
2638
2639   def int_x86_avx512_ktestc_b :
2640           Intrinsic<[llvm_i32_ty], [llvm_v8i1_ty, llvm_v8i1_ty], [IntrNoMem]>;
2641   def int_x86_avx512_ktestc_w :
2642           Intrinsic<[llvm_i32_ty], [llvm_v16i1_ty, llvm_v16i1_ty], [IntrNoMem]>;
2643   def int_x86_avx512_ktestc_d :
2644           Intrinsic<[llvm_i32_ty], [llvm_v32i1_ty, llvm_v32i1_ty], [IntrNoMem]>;
2645   def int_x86_avx512_ktestc_q :
2646           Intrinsic<[llvm_i32_ty], [llvm_v64i1_ty, llvm_v64i1_ty], [IntrNoMem]>;
2647
2648   def int_x86_avx512_ktestz_b :
2649           Intrinsic<[llvm_i32_ty], [llvm_v8i1_ty, llvm_v8i1_ty], [IntrNoMem]>;
2650   def int_x86_avx512_ktestz_w :
2651           Intrinsic<[llvm_i32_ty], [llvm_v16i1_ty, llvm_v16i1_ty], [IntrNoMem]>;
2652   def int_x86_avx512_ktestz_d :
2653           Intrinsic<[llvm_i32_ty], [llvm_v32i1_ty, llvm_v32i1_ty], [IntrNoMem]>;
2654   def int_x86_avx512_ktestz_q :
2655           Intrinsic<[llvm_i32_ty], [llvm_v64i1_ty, llvm_v64i1_ty], [IntrNoMem]>;
2656 }
2657
2658 // Conversion ops
2659 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2660   def int_x86_avx512_cvttss2si : GCCBuiltin<"__builtin_ia32_vcvttss2si32">,
2661               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty],
2662                         [IntrNoMem, ImmArg<1>]>;
2663   def int_x86_avx512_cvttss2si64 : GCCBuiltin<"__builtin_ia32_vcvttss2si64">,
2664               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty],
2665                         [IntrNoMem, ImmArg<1>]>;
2666   def int_x86_avx512_cvttss2usi : GCCBuiltin<"__builtin_ia32_vcvttss2usi32">,
2667               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty],
2668                         [IntrNoMem, ImmArg<1>]>;
2669   def int_x86_avx512_cvttss2usi64 : GCCBuiltin<"__builtin_ia32_vcvttss2usi64">,
2670               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty],
2671                         [IntrNoMem, ImmArg<1>]>;
2672   def int_x86_avx512_cvtusi2ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss32">,
2673               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
2674                          llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2675   def int_x86_avx512_cvtusi642ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss64">,
2676               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
2677                          llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2678   def int_x86_avx512_cvttsd2si : GCCBuiltin<"__builtin_ia32_vcvttsd2si32">,
2679               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty],
2680                         [IntrNoMem, ImmArg<1>]>;
2681   def int_x86_avx512_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_vcvttsd2si64">,
2682               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty],
2683                         [IntrNoMem, ImmArg<1>]>;
2684   def int_x86_avx512_cvttsd2usi : GCCBuiltin<"__builtin_ia32_vcvttsd2usi32">,
2685               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty],
2686                         [IntrNoMem, ImmArg<1>]>;
2687   def int_x86_avx512_cvttsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvttsd2usi64">,
2688               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty],
2689                         [IntrNoMem, ImmArg<1>]>;
2690   def int_x86_avx512_cvtusi642sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd64">,
2691               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
2692                          llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2693   def int_x86_avx512_vcvtss2usi32 : GCCBuiltin<"__builtin_ia32_vcvtss2usi32">,
2694               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty],
2695                         [IntrNoMem, ImmArg<1>]>;
2696   def int_x86_avx512_vcvtss2usi64 : GCCBuiltin<"__builtin_ia32_vcvtss2usi64">,
2697               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty],
2698                         [IntrNoMem, ImmArg<1>]>;
2699   def int_x86_avx512_vcvtss2si32 : GCCBuiltin<"__builtin_ia32_vcvtss2si32">,
2700               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty],
2701                         [IntrNoMem, ImmArg<1>]>;
2702   def int_x86_avx512_vcvtss2si64 : GCCBuiltin<"__builtin_ia32_vcvtss2si64">,
2703               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty],
2704                         [IntrNoMem, ImmArg<1>]>;
2705   def int_x86_avx512_vcvtsd2usi32 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi32">,
2706               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty],
2707                         [IntrNoMem, ImmArg<1>]>;
2708   def int_x86_avx512_vcvtsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi64">,
2709               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty],
2710                         [IntrNoMem, ImmArg<1>]>;
2711   def int_x86_avx512_vcvtsd2si32 : GCCBuiltin<"__builtin_ia32_vcvtsd2si32">,
2712               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty],
2713                         [IntrNoMem, ImmArg<1>]>;
2714   def int_x86_avx512_vcvtsd2si64 : GCCBuiltin<"__builtin_ia32_vcvtsd2si64">,
2715               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty],
2716                         [IntrNoMem, ImmArg<1>]>;
2717   def int_x86_avx512_cvtsi2ss32 : GCCBuiltin<"__builtin_ia32_cvtsi2ss32">,
2718               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
2719                          llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2720   def int_x86_avx512_cvtsi2ss64 : GCCBuiltin<"__builtin_ia32_cvtsi2ss64">,
2721               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
2722                          llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2723   def int_x86_avx512_cvtsi2sd64 : GCCBuiltin<"__builtin_ia32_cvtsi2sd64">,
2724               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
2725                          llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
2726 }
2727
2728 // Pack ops.
2729 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2730   def int_x86_avx512_packsswb_512 : GCCBuiltin<"__builtin_ia32_packsswb512">,
2731               Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty],
2732                         [IntrNoMem]>;
2733   def int_x86_avx512_packssdw_512 : GCCBuiltin<"__builtin_ia32_packssdw512">,
2734               Intrinsic<[llvm_v32i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty],
2735                          [IntrNoMem]>;
2736   def int_x86_avx512_packuswb_512 : GCCBuiltin<"__builtin_ia32_packuswb512">,
2737               Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty],
2738                          [IntrNoMem]>;
2739   def int_x86_avx512_packusdw_512 : GCCBuiltin<"__builtin_ia32_packusdw512">,
2740               Intrinsic<[llvm_v32i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty],
2741                          [IntrNoMem]>;
2742 }
2743
2744 // Vector convert
2745 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
2746   def int_x86_avx512_sitofp_round :
2747           Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
2748                     [IntrNoMem, ImmArg<1>]>;
2749
2750   def int_x86_avx512_uitofp_round :
2751           Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
2752                     [IntrNoMem, ImmArg<1>]>;
2753
2754   def int_x86_avx512_mask_cvtpd2dq_128 :
2755         GCCBuiltin<"__builtin_ia32_cvtpd2dq128_mask">,
2756           Intrinsic<[llvm_v4i32_ty],
2757           [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
2758           [IntrNoMem]>;
2759
2760   def int_x86_avx512_mask_cvtpd2dq_512 :
2761         GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">,
2762           Intrinsic<[llvm_v8i32_ty],
2763           [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
2764           [IntrNoMem, ImmArg<3>]>;
2765
2766   def int_x86_avx512_mask_cvtpd2ps_512 :
2767         GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">,
2768           Intrinsic<[llvm_v8f32_ty],
2769           [llvm_v8f64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
2770           [IntrNoMem, ImmArg<3>]>;
2771
2772   def int_x86_avx512_mask_cvtsd2ss_round :
2773         GCCBuiltin<"__builtin_ia32_cvtsd2ss_round_mask">,
2774           Intrinsic<[llvm_v4f32_ty],
2775           [llvm_v4f32_ty, llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty],
2776           [IntrNoMem, ImmArg<4>]>;
2777
2778   def int_x86_avx512_mask_cvtss2sd_round :
2779         GCCBuiltin<"__builtin_ia32_cvtss2sd_round_mask">,
2780           Intrinsic<[llvm_v2f64_ty],
2781           [llvm_v2f64_ty, llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty],
2782           [IntrNoMem, ImmArg<4>]>;
2783
2784   def int_x86_avx512_mask_cvtpd2ps :
2785         GCCBuiltin<"__builtin_ia32_cvtpd2ps_mask">,
2786           Intrinsic<[llvm_v4f32_ty],
2787           [llvm_v2f64_ty, llvm_v4f32_ty,  llvm_i8_ty],
2788           [IntrNoMem]>;
2789
2790   def int_x86_avx512_mask_cvtpd2qq_128 :
2791         GCCBuiltin<"__builtin_ia32_cvtpd2qq128_mask">,
2792           Intrinsic<[llvm_v2i64_ty],
2793           [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
2794           [IntrNoMem]>;
2795
2796   def int_x86_avx512_mask_cvtpd2qq_256 :
2797         GCCBuiltin<"__builtin_ia32_cvtpd2qq256_mask">,
2798           Intrinsic<[llvm_v4i64_ty],
2799           [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
2800           [IntrNoMem]>;
2801
2802   def int_x86_avx512_mask_cvtpd2qq_512 :
2803         GCCBuiltin<"__builtin_ia32_cvtpd2qq512_mask">,
2804           Intrinsic<[llvm_v8i64_ty],
2805           [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
2806           [IntrNoMem, ImmArg<3>]>;
2807
2808   def int_x86_avx512_mask_cvtpd2udq_128 :
2809         GCCBuiltin<"__builtin_ia32_cvtpd2udq128_mask">,
2810           Intrinsic<[llvm_v4i32_ty],
2811           [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
2812           [IntrNoMem]>;
2813
2814   def int_x86_avx512_mask_cvtpd2udq_256 :
2815         GCCBuiltin<"__builtin_ia32_cvtpd2udq256_mask">,
2816           Intrinsic<[llvm_v4i32_ty],
2817           [llvm_v4f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
2818           [IntrNoMem]>;
2819
2820   def int_x86_avx512_mask_cvtpd2udq_512 :
2821         GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">,
2822           Intrinsic<[llvm_v8i32_ty],
2823           [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
2824           [IntrNoMem, ImmArg<3>]>;
2825
2826   def int_x86_avx512_mask_cvtpd2uqq_128 :
2827         GCCBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">,
2828           Intrinsic<[llvm_v2i64_ty],
2829           [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
2830           [IntrNoMem]>;
2831
2832   def int_x86_avx512_mask_cvtpd2uqq_256 :
2833         GCCBuiltin<"__builtin_ia32_cvtpd2uqq256_mask">,
2834           Intrinsic<[llvm_v4i64_ty],
2835           [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
2836           [IntrNoMem]>;
2837
2838   def int_x86_avx512_mask_cvtpd2uqq_512 :
2839         GCCBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">,
2840           Intrinsic<[llvm_v8i64_ty],
2841           [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
2842           [IntrNoMem, ImmArg<3>]>;
2843
2844   def int_x86_avx512_mask_cvtps2dq_128 :
2845         GCCBuiltin<"__builtin_ia32_cvtps2dq128_mask">,
2846           Intrinsic<[llvm_v4i32_ty],
2847           [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
2848           [IntrNoMem]>;
2849
2850   def int_x86_avx512_mask_cvtps2dq_256 :
2851         GCCBuiltin<"__builtin_ia32_cvtps2dq256_mask">,
2852           Intrinsic<[llvm_v8i32_ty],
2853           [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
2854           [IntrNoMem]>;
2855
2856   def int_x86_avx512_mask_cvtps2dq_512 :
2857         GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">,
2858           Intrinsic<[llvm_v16i32_ty],
2859           [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
2860           [IntrNoMem, ImmArg<3>]>;
2861
2862   def int_x86_avx512_mask_cvtps2pd_512 :
2863         GCCBuiltin<"__builtin_ia32_cvtps2pd512_mask">,
2864           Intrinsic<[llvm_v8f64_ty],
2865           [llvm_v8f32_ty, llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
2866           [IntrNoMem, ImmArg<3>]>;
2867
2868   def int_x86_avx512_mask_cvtps2qq_128 :
2869         GCCBuiltin<"__builtin_ia32_cvtps2qq128_mask">,
2870           Intrinsic<[llvm_v2i64_ty],
2871           [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
2872           [IntrNoMem]>;
2873
2874   def int_x86_avx512_mask_cvtps2qq_256 :
2875         GCCBuiltin<"__builtin_ia32_cvtps2qq256_mask">,
2876           Intrinsic<[llvm_v4i64_ty],
2877           [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
2878           [IntrNoMem]>;
2879
2880   def int_x86_avx512_mask_cvtps2qq_512 :
2881         GCCBuiltin<"__builtin_ia32_cvtps2qq512_mask">,
2882           Intrinsic<[llvm_v8i64_ty],
2883           [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
2884           [IntrNoMem, ImmArg<3>]>;
2885
2886   def int_x86_avx512_mask_cvtps2udq_128 :
2887         GCCBuiltin<"__builtin_ia32_cvtps2udq128_mask">,
2888           Intrinsic<[llvm_v4i32_ty],
2889           [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
2890           [IntrNoMem]>;
2891
2892   def int_x86_avx512_mask_cvtps2udq_256 :
2893         GCCBuiltin<"__builtin_ia32_cvtps2udq256_mask">,
2894           Intrinsic<[llvm_v8i32_ty],
2895           [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
2896           [IntrNoMem]>;
2897
2898   def int_x86_avx512_mask_cvtps2udq_512 :
2899         GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">,
2900           Intrinsic<[llvm_v16i32_ty],
2901           [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
2902           [IntrNoMem, ImmArg<3>]>;
2903
2904   def int_x86_avx512_mask_cvtps2uqq_128 :
2905         GCCBuiltin<"__builtin_ia32_cvtps2uqq128_mask">,
2906           Intrinsic<[llvm_v2i64_ty],
2907           [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
2908           [IntrNoMem]>;
2909
2910   def int_x86_avx512_mask_cvtps2uqq_256 :
2911         GCCBuiltin<"__builtin_ia32_cvtps2uqq256_mask">,
2912           Intrinsic<[llvm_v4i64_ty],
2913           [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
2914           [IntrNoMem]>;
2915
2916   def int_x86_avx512_mask_cvtps2uqq_512 :
2917         GCCBuiltin<"__builtin_ia32_cvtps2uqq512_mask">,
2918           Intrinsic<[llvm_v8i64_ty],
2919           [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
2920           [IntrNoMem, ImmArg<3>]>;
2921
2922   def int_x86_avx512_mask_cvtqq2ps_128 :
2923         GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">,
2924           Intrinsic<[llvm_v4f32_ty],
2925           [llvm_v2i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
2926           [IntrNoMem]>;
2927
2928   def int_x86_avx512_mask_cvttpd2dq_128 :
2929         GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">,
2930           Intrinsic<[llvm_v4i32_ty],
2931           [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
2932           [IntrNoMem]>;
2933
2934   def int_x86_avx512_mask_cvttpd2dq_512 :
2935         GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">,
2936           Intrinsic<[llvm_v8i32_ty],
2937           [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
2938           [IntrNoMem, ImmArg<3>]>;
2939
2940   def int_x86_avx512_mask_cvttpd2qq_128 :
2941         GCCBuiltin<"__builtin_ia32_cvttpd2qq128_mask">,
2942           Intrinsic<[llvm_v2i64_ty],
2943           [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
2944           [IntrNoMem]>;
2945
2946   def int_x86_avx512_mask_cvttpd2qq_256 :
2947         GCCBuiltin<"__builtin_ia32_cvttpd2qq256_mask">,
2948           Intrinsic<[llvm_v4i64_ty],
2949           [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
2950           [IntrNoMem]>;
2951
2952   def int_x86_avx512_mask_cvttpd2qq_512 :
2953         GCCBuiltin<"__builtin_ia32_cvttpd2qq512_mask">,
2954           Intrinsic<[llvm_v8i64_ty],
2955           [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
2956           [IntrNoMem, ImmArg<3>]>;
2957
2958   def int_x86_avx512_mask_cvttpd2udq_128 :
2959         GCCBuiltin<"__builtin_ia32_cvttpd2udq128_mask">,
2960           Intrinsic<[llvm_v4i32_ty],
2961           [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
2962           [IntrNoMem]>;
2963
2964   def int_x86_avx512_mask_cvttpd2udq_256 :
2965         GCCBuiltin<"__builtin_ia32_cvttpd2udq256_mask">,
2966           Intrinsic<[llvm_v4i32_ty],
2967           [llvm_v4f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
2968           [IntrNoMem]>;
2969
2970   def int_x86_avx512_mask_cvttpd2udq_512 :
2971         GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">,
2972           Intrinsic<[llvm_v8i32_ty],
2973           [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
2974           [IntrNoMem, ImmArg<3>]>;
2975
2976   def int_x86_avx512_mask_cvttpd2uqq_128 :
2977         GCCBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">,
2978           Intrinsic<[llvm_v2i64_ty],
2979           [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
2980           [IntrNoMem]>;
2981
2982   def int_x86_avx512_mask_cvttpd2uqq_256 :
2983         GCCBuiltin<"__builtin_ia32_cvttpd2uqq256_mask">,
2984           Intrinsic<[llvm_v4i64_ty],
2985           [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
2986           [IntrNoMem]>;
2987
2988   def int_x86_avx512_mask_cvttpd2uqq_512 :
2989         GCCBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">,
2990           Intrinsic<[llvm_v8i64_ty],
2991           [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
2992           [IntrNoMem, ImmArg<3>]>;
2993
2994   def int_x86_avx512_mask_cvttps2dq_512 :
2995         GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">,
2996           Intrinsic<[llvm_v16i32_ty],
2997           [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
2998           [IntrNoMem, ImmArg<3>]>;
2999
3000   def int_x86_avx512_mask_cvttps2qq_128 :
3001         GCCBuiltin<"__builtin_ia32_cvttps2qq128_mask">,
3002           Intrinsic<[llvm_v2i64_ty],
3003           [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
3004           [IntrNoMem]>;
3005
3006   def int_x86_avx512_mask_cvttps2qq_256 :
3007         GCCBuiltin<"__builtin_ia32_cvttps2qq256_mask">,
3008           Intrinsic<[llvm_v4i64_ty],
3009           [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
3010           [IntrNoMem]>;
3011
3012   def int_x86_avx512_mask_cvttps2qq_512 :
3013         GCCBuiltin<"__builtin_ia32_cvttps2qq512_mask">,
3014           Intrinsic<[llvm_v8i64_ty],
3015           [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
3016           [IntrNoMem, ImmArg<3>]>;
3017
3018   def int_x86_avx512_mask_cvttps2udq_128 :
3019         GCCBuiltin<"__builtin_ia32_cvttps2udq128_mask">,
3020           Intrinsic<[llvm_v4i32_ty],
3021           [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
3022           [IntrNoMem]>;
3023
3024   def int_x86_avx512_mask_cvttps2udq_256 :
3025         GCCBuiltin<"__builtin_ia32_cvttps2udq256_mask">,
3026           Intrinsic<[llvm_v8i32_ty],
3027           [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
3028           [IntrNoMem]>;
3029
3030   def int_x86_avx512_mask_cvttps2udq_512 :
3031         GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">,
3032           Intrinsic<[llvm_v16i32_ty],
3033           [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
3034           [IntrNoMem, ImmArg<3>]>;
3035
3036   def int_x86_avx512_mask_cvttps2uqq_128 :
3037         GCCBuiltin<"__builtin_ia32_cvttps2uqq128_mask">,
3038           Intrinsic<[llvm_v2i64_ty],
3039           [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
3040           [IntrNoMem]>;
3041
3042   def int_x86_avx512_mask_cvttps2uqq_256 :
3043         GCCBuiltin<"__builtin_ia32_cvttps2uqq256_mask">,
3044           Intrinsic<[llvm_v4i64_ty],
3045           [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
3046           [IntrNoMem]>;
3047
3048   def int_x86_avx512_mask_cvttps2uqq_512 :
3049         GCCBuiltin<"__builtin_ia32_cvttps2uqq512_mask">,
3050           Intrinsic<[llvm_v8i64_ty],
3051           [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
3052           [IntrNoMem, ImmArg<3>]>;
3053
3054   def int_x86_avx512_mask_cvtuqq2ps_128 :
3055         GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">,
3056           Intrinsic<[llvm_v4f32_ty],
3057           [llvm_v2i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
3058           [IntrNoMem]>;
3059
3060   def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">,
3061         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty,
3062                                      llvm_v2f64_ty,  llvm_i8_ty],
3063                   [IntrNoMem, ImmArg<1>]>;
3064   def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">,
3065         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty,
3066                                      llvm_v4f64_ty,  llvm_i8_ty],
3067                   [IntrNoMem, ImmArg<1>]>;
3068   def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">,
3069         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty,
3070                                      llvm_i8_ty, llvm_i32_ty],
3071                   [IntrNoMem, ImmArg<1>, ImmArg<4>]>;
3072   def int_x86_avx512_mask_rndscale_ps_128 : GCCBuiltin<"__builtin_ia32_rndscaleps_128_mask">,
3073         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty,
3074                                      llvm_v4f32_ty,  llvm_i8_ty],
3075                   [IntrNoMem, ImmArg<1>]>;
3076   def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">,
3077         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty,
3078                                      llvm_v8f32_ty,  llvm_i8_ty],
3079                   [IntrNoMem, ImmArg<1>]>;
3080   def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">,
3081         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty,
3082                                      llvm_i16_ty, llvm_i32_ty],
3083                   [IntrNoMem, ImmArg<1>, ImmArg<4>]>;
3084   def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">,
3085         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty,
3086                                      llvm_v2f64_ty,  llvm_i8_ty],
3087                   [IntrNoMem, ImmArg<1>]>;
3088   def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">,
3089         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty,
3090                                      llvm_v4f64_ty,  llvm_i8_ty],
3091                   [IntrNoMem, ImmArg<1>]>;
3092   def int_x86_avx512_mask_reduce_pd_512 : GCCBuiltin<"__builtin_ia32_reducepd512_mask">,
3093         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty,
3094                                      llvm_i8_ty, llvm_i32_ty],
3095                   [IntrNoMem, ImmArg<1>, ImmArg<4>]>;
3096   def int_x86_avx512_mask_reduce_ps_128 : GCCBuiltin<"__builtin_ia32_reduceps128_mask">,
3097         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty,
3098                                      llvm_v4f32_ty,  llvm_i8_ty],
3099                   [IntrNoMem, ImmArg<1>]>;
3100   def int_x86_avx512_mask_reduce_ps_256 : GCCBuiltin<"__builtin_ia32_reduceps256_mask">,
3101         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty,
3102                                      llvm_v8f32_ty,  llvm_i8_ty],
3103                   [IntrNoMem, ImmArg<1>]>;
3104   def int_x86_avx512_mask_reduce_ps_512 : GCCBuiltin<"__builtin_ia32_reduceps512_mask">,
3105         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty,
3106                                      llvm_i16_ty, llvm_i32_ty],
3107                   [IntrNoMem, ImmArg<1>, ImmArg<4>]>;
3108 def int_x86_avx512_mask_range_pd_128 : GCCBuiltin<"__builtin_ia32_rangepd128_mask">,
3109         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty,
3110                                     llvm_v2f64_ty,  llvm_i8_ty],
3111                   [IntrNoMem, ImmArg<2>]>;
3112 def int_x86_avx512_mask_range_pd_256 : GCCBuiltin<"__builtin_ia32_rangepd256_mask">,
3113         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty,
3114                                     llvm_v4f64_ty,  llvm_i8_ty],
3115                   [IntrNoMem, ImmArg<2>]>;
3116 def int_x86_avx512_mask_range_pd_512 : GCCBuiltin<"__builtin_ia32_rangepd512_mask">,
3117         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty,
3118                                     llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
3119                   [IntrNoMem, ImmArg<2>, ImmArg<5>]>;
3120 def int_x86_avx512_mask_range_ps_128 : GCCBuiltin<"__builtin_ia32_rangeps128_mask">,
3121         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty,
3122                                     llvm_v4f32_ty,  llvm_i8_ty],
3123                   [IntrNoMem, ImmArg<2>]>;
3124 def int_x86_avx512_mask_range_ps_256 : GCCBuiltin<"__builtin_ia32_rangeps256_mask">,
3125         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty,
3126                                     llvm_v8f32_ty,  llvm_i8_ty],
3127                   [IntrNoMem, ImmArg<2>]>;
3128 def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mask">,
3129         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty,
3130                                      llvm_v16f32_ty,  llvm_i16_ty,  llvm_i32_ty],
3131                   [IntrNoMem, ImmArg<2>, ImmArg<5>]>;
3132 }
3133
3134 // Vector load with broadcast
3135 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
3136    def int_x86_avx512_broadcastmw_512 :
3137           GCCBuiltin<"__builtin_ia32_broadcastmw512">,
3138           Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>;
3139    def int_x86_avx512_broadcastmw_256 :
3140           GCCBuiltin<"__builtin_ia32_broadcastmw256">,
3141           Intrinsic<[llvm_v8i32_ty], [llvm_i16_ty], [IntrNoMem]>;
3142    def int_x86_avx512_broadcastmw_128 :
3143           GCCBuiltin<"__builtin_ia32_broadcastmw128">,
3144           Intrinsic<[llvm_v4i32_ty], [llvm_i16_ty], [IntrNoMem]>;
3145    def int_x86_avx512_broadcastmb_512 :
3146           GCCBuiltin<"__builtin_ia32_broadcastmb512">,
3147           Intrinsic<[llvm_v8i64_ty], [llvm_i8_ty], [IntrNoMem]>;
3148    def int_x86_avx512_broadcastmb_256 :
3149           GCCBuiltin<"__builtin_ia32_broadcastmb256">,
3150           Intrinsic<[llvm_v4i64_ty], [llvm_i8_ty], [IntrNoMem]>;
3151    def int_x86_avx512_broadcastmb_128 :
3152           GCCBuiltin<"__builtin_ia32_broadcastmb128">,
3153           Intrinsic<[llvm_v2i64_ty], [llvm_i8_ty], [IntrNoMem]>;
3154 }
3155
3156 // Arithmetic ops
3157 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
3158
3159   def int_x86_avx512_add_ps_512 : GCCBuiltin<"__builtin_ia32_addps512">,
3160           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3161                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3162   def int_x86_avx512_add_pd_512 : GCCBuiltin<"__builtin_ia32_addpd512">,
3163           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3164                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3165   def int_x86_avx512_sub_ps_512 : GCCBuiltin<"__builtin_ia32_subps512">,
3166           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3167                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3168   def int_x86_avx512_sub_pd_512 : GCCBuiltin<"__builtin_ia32_subpd512">,
3169           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3170                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3171   def int_x86_avx512_mul_ps_512 : GCCBuiltin<"__builtin_ia32_mulps512">,
3172           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3173                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3174   def int_x86_avx512_mul_pd_512 : GCCBuiltin<"__builtin_ia32_mulpd512">,
3175           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3176                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3177   def int_x86_avx512_div_ps_512 : GCCBuiltin<"__builtin_ia32_divps512">,
3178           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3179                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3180   def int_x86_avx512_div_pd_512 : GCCBuiltin<"__builtin_ia32_divpd512">,
3181           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3182                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3183
3184   def int_x86_avx512_max_ps_512 : GCCBuiltin<"__builtin_ia32_maxps512">,
3185           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3186                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3187   def int_x86_avx512_max_pd_512 : GCCBuiltin<"__builtin_ia32_maxpd512">,
3188           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3189                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3190   def int_x86_avx512_min_ps_512 : GCCBuiltin<"__builtin_ia32_minps512">,
3191           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3192                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3193   def int_x86_avx512_min_pd_512 : GCCBuiltin<"__builtin_ia32_minpd512">,
3194           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3195                      llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
3196
3197   def int_x86_avx512_mask_add_ss_round : GCCBuiltin<"__builtin_ia32_addss_round_mask">,
3198           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3199                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3200   def int_x86_avx512_mask_div_ss_round : GCCBuiltin<"__builtin_ia32_divss_round_mask">,
3201           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3202                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3203   def int_x86_avx512_mask_mul_ss_round : GCCBuiltin<"__builtin_ia32_mulss_round_mask">,
3204           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3205                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3206   def int_x86_avx512_mask_sub_ss_round : GCCBuiltin<"__builtin_ia32_subss_round_mask">,
3207           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3208                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3209   def int_x86_avx512_mask_max_ss_round : GCCBuiltin<"__builtin_ia32_maxss_round_mask">,
3210           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3211                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3212   def int_x86_avx512_mask_min_ss_round : GCCBuiltin<"__builtin_ia32_minss_round_mask">,
3213           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3214                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3215   def int_x86_avx512_mask_add_sd_round : GCCBuiltin<"__builtin_ia32_addsd_round_mask">,
3216           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3217                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3218   def int_x86_avx512_mask_div_sd_round : GCCBuiltin<"__builtin_ia32_divsd_round_mask">,
3219           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3220                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3221   def int_x86_avx512_mask_mul_sd_round : GCCBuiltin<"__builtin_ia32_mulsd_round_mask">,
3222           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3223                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3224   def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_round_mask">,
3225           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3226                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3227   def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_round_mask">,
3228           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3229                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3230   def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_round_mask">,
3231           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3232                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>;
3233
3234   def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round_mask">,
3235           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3236                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
3237                                      [IntrNoMem, ImmArg<4>, ImmArg<5>]>;
3238   def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round_mask">,
3239           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3240                                       llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
3241                                      [IntrNoMem, ImmArg<4>, ImmArg<5>]>;
3242   def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round_mask">,
3243           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3244                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
3245                                      [IntrNoMem, ImmArg<4>, ImmArg<5>]>;
3246   def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round_mask">,
3247           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3248                                       llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
3249                                      [IntrNoMem, ImmArg<4>, ImmArg<5>]>;
3250   def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess_mask">,
3251           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3252                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
3253                                      [IntrNoMem, ImmArg<4>, ImmArg<5>]>;
3254   def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd_mask">,
3255           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3256                                       llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
3257                                      [IntrNoMem, ImmArg<4>, ImmArg<5>]>;
3258   def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round_mask">,
3259           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3260                                       llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty],
3261                                      [IntrNoMem, ImmArg<4>]>;
3262   def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round_mask">,
3263           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3264                                       llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty],
3265                                      [IntrNoMem, ImmArg<4>]>;
3266   def int_x86_avx512_mask_scalef_pd_128 : GCCBuiltin<"__builtin_ia32_scalefpd128_mask">,
3267           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3268                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
3269   def int_x86_avx512_mask_scalef_pd_256 : GCCBuiltin<"__builtin_ia32_scalefpd256_mask">,
3270           Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
3271                     llvm_v4f64_ty, llvm_i8_ty],[IntrNoMem]>;
3272   def int_x86_avx512_mask_scalef_pd_512 : GCCBuiltin<"__builtin_ia32_scalefpd512_mask">,
3273           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3274                                       llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty],
3275                     [IntrNoMem, ImmArg<4>]>;
3276   def int_x86_avx512_mask_scalef_ps_128 : GCCBuiltin<"__builtin_ia32_scalefps128_mask">,
3277           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3278                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
3279   def int_x86_avx512_mask_scalef_ps_256 : GCCBuiltin<"__builtin_ia32_scalefps256_mask">,
3280           Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
3281                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
3282   def int_x86_avx512_mask_scalef_ps_512 : GCCBuiltin<"__builtin_ia32_scalefps512_mask">,
3283           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3284                                        llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty],
3285                     [IntrNoMem, ImmArg<4>]>;
3286
3287   def int_x86_avx512_mask_sqrt_ss :
3288         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3289                                     llvm_i8_ty, llvm_i32_ty],
3290                   [IntrNoMem, ImmArg<4>]>;
3291   def int_x86_avx512_mask_sqrt_sd :
3292         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3293                                     llvm_i8_ty, llvm_i32_ty],
3294                   [IntrNoMem, ImmArg<4>]>;
3295
3296   def int_x86_avx512_sqrt_pd_512 :
3297         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty],
3298                   [IntrNoMem, ImmArg<1>]>;
3299   def int_x86_avx512_sqrt_ps_512 :
3300         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty],
3301                   [IntrNoMem, ImmArg<1>]>;
3302   def int_x86_avx512_mask_fixupimm_pd_128 :
3303          GCCBuiltin<"__builtin_ia32_fixupimmpd128_mask">,
3304           Intrinsic<[llvm_v2f64_ty],
3305           [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty],
3306           [IntrNoMem, ImmArg<3>]>;
3307   def int_x86_avx512_maskz_fixupimm_pd_128 :
3308          GCCBuiltin<"__builtin_ia32_fixupimmpd128_maskz">,
3309           Intrinsic<[llvm_v2f64_ty],
3310           [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty],
3311           [IntrNoMem, ImmArg<3>]>;
3312   def int_x86_avx512_mask_fixupimm_pd_256 :
3313          GCCBuiltin<"__builtin_ia32_fixupimmpd256_mask">,
3314           Intrinsic<[llvm_v4f64_ty],
3315           [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty],
3316           [IntrNoMem, ImmArg<3>]>;
3317   def int_x86_avx512_maskz_fixupimm_pd_256 :
3318          GCCBuiltin<"__builtin_ia32_fixupimmpd256_maskz">,
3319           Intrinsic<[llvm_v4f64_ty],
3320           [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty],
3321           [IntrNoMem, ImmArg<3>]>;
3322   def int_x86_avx512_mask_fixupimm_pd_512 :
3323          GCCBuiltin<"__builtin_ia32_fixupimmpd512_mask">,
3324           Intrinsic<[llvm_v8f64_ty],
3325           [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty,
3326           llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3327   def int_x86_avx512_maskz_fixupimm_pd_512 :
3328          GCCBuiltin<"__builtin_ia32_fixupimmpd512_maskz">,
3329           Intrinsic<[llvm_v8f64_ty],
3330           [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty,
3331           llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3332   def int_x86_avx512_mask_fixupimm_ps_128 :
3333          GCCBuiltin<"__builtin_ia32_fixupimmps128_mask">,
3334           Intrinsic<[llvm_v4f32_ty],
3335           [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty],
3336           [IntrNoMem, ImmArg<3>]>;
3337   def int_x86_avx512_maskz_fixupimm_ps_128 :
3338          GCCBuiltin<"__builtin_ia32_fixupimmps128_maskz">,
3339           Intrinsic<[llvm_v4f32_ty],
3340           [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty],
3341           [IntrNoMem, ImmArg<3>]>;
3342   def int_x86_avx512_mask_fixupimm_ps_256 :
3343          GCCBuiltin<"__builtin_ia32_fixupimmps256_mask">,
3344           Intrinsic<[llvm_v8f32_ty],
3345           [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty],
3346           [IntrNoMem, ImmArg<3>]>;
3347   def int_x86_avx512_maskz_fixupimm_ps_256 :
3348          GCCBuiltin<"__builtin_ia32_fixupimmps256_maskz">,
3349           Intrinsic<[llvm_v8f32_ty],
3350           [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty],
3351           [IntrNoMem, ImmArg<3>]>;
3352   def int_x86_avx512_mask_fixupimm_ps_512 :
3353          GCCBuiltin<"__builtin_ia32_fixupimmps512_mask">,
3354           Intrinsic<[llvm_v16f32_ty],
3355           [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty,
3356           llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3357   def int_x86_avx512_maskz_fixupimm_ps_512 :
3358          GCCBuiltin<"__builtin_ia32_fixupimmps512_maskz">,
3359           Intrinsic<[llvm_v16f32_ty],
3360           [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty,
3361           llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3362   def int_x86_avx512_mask_fixupimm_sd :
3363          GCCBuiltin<"__builtin_ia32_fixupimmsd_mask">,
3364           Intrinsic<[llvm_v2f64_ty],
3365           [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty,
3366           llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3367   def int_x86_avx512_maskz_fixupimm_sd :
3368          GCCBuiltin<"__builtin_ia32_fixupimmsd_maskz">,
3369           Intrinsic<[llvm_v2f64_ty],
3370           [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty,
3371           llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3372   def int_x86_avx512_mask_fixupimm_ss :
3373          GCCBuiltin<"__builtin_ia32_fixupimmss_mask">,
3374           Intrinsic<[llvm_v4f32_ty],
3375           [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty,
3376           llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3377   def int_x86_avx512_maskz_fixupimm_ss :
3378          GCCBuiltin<"__builtin_ia32_fixupimmss_maskz">,
3379           Intrinsic<[llvm_v4f32_ty],
3380           [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty,
3381           llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>;
3382   def int_x86_avx512_mask_getexp_pd_128 : GCCBuiltin<"__builtin_ia32_getexppd128_mask">,
3383         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3384                                     llvm_i8_ty], [IntrNoMem]>;
3385   def int_x86_avx512_mask_getexp_pd_256 : GCCBuiltin<"__builtin_ia32_getexppd256_mask">,
3386         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
3387                                     llvm_i8_ty], [IntrNoMem]>;
3388   def int_x86_avx512_mask_getexp_pd_512 : GCCBuiltin<"__builtin_ia32_getexppd512_mask">,
3389         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3390                                     llvm_i8_ty, llvm_i32_ty],
3391                   [IntrNoMem, ImmArg<3>]>;
3392   def int_x86_avx512_mask_getexp_ps_128 : GCCBuiltin<"__builtin_ia32_getexpps128_mask">,
3393         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3394                                      llvm_i8_ty], [IntrNoMem]>;
3395   def int_x86_avx512_mask_getexp_ps_256 : GCCBuiltin<"__builtin_ia32_getexpps256_mask">,
3396         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
3397                                      llvm_i8_ty], [IntrNoMem]>;
3398   def int_x86_avx512_mask_getexp_ps_512 : GCCBuiltin<"__builtin_ia32_getexpps512_mask">,
3399         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3400                                      llvm_i16_ty, llvm_i32_ty],
3401                   [IntrNoMem, ImmArg<3>]>;
3402
3403   def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss128_round_mask">,
3404         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3405                                     llvm_i8_ty, llvm_i32_ty],
3406                   [IntrNoMem, ImmArg<4>]>;
3407   def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd128_round_mask">,
3408         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3409                                     llvm_i8_ty, llvm_i32_ty],
3410                   [IntrNoMem, ImmArg<4>]>;
3411
3412   def int_x86_avx512_mask_getmant_pd_128 :
3413          GCCBuiltin<"__builtin_ia32_getmantpd128_mask">,
3414           Intrinsic<[llvm_v2f64_ty],
3415           [llvm_v2f64_ty,llvm_i32_ty, llvm_v2f64_ty,  llvm_i8_ty],
3416           [IntrNoMem, ImmArg<1>]>;
3417
3418   def int_x86_avx512_mask_getmant_pd_256 :
3419          GCCBuiltin<"__builtin_ia32_getmantpd256_mask">,
3420           Intrinsic<[llvm_v4f64_ty],
3421           [llvm_v4f64_ty,llvm_i32_ty, llvm_v4f64_ty,  llvm_i8_ty],
3422           [IntrNoMem, ImmArg<1>]>;
3423
3424   def int_x86_avx512_mask_getmant_pd_512 :
3425          GCCBuiltin<"__builtin_ia32_getmantpd512_mask">,
3426           Intrinsic<[llvm_v8f64_ty],
3427           [llvm_v8f64_ty,llvm_i32_ty, llvm_v8f64_ty,  llvm_i8_ty,llvm_i32_ty ],
3428           [IntrNoMem, ImmArg<1>, ImmArg<4>]>;
3429
3430   def int_x86_avx512_mask_getmant_ps_128 :
3431          GCCBuiltin<"__builtin_ia32_getmantps128_mask">,
3432           Intrinsic<[llvm_v4f32_ty],
3433           [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty,  llvm_i8_ty],
3434           [IntrNoMem, ImmArg<1>]>;
3435
3436   def int_x86_avx512_mask_getmant_ps_256 :
3437          GCCBuiltin<"__builtin_ia32_getmantps256_mask">,
3438           Intrinsic<[llvm_v8f32_ty],
3439           [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty,  llvm_i8_ty],
3440           [IntrNoMem, ImmArg<1>]>;
3441
3442   def int_x86_avx512_mask_getmant_ps_512 :
3443          GCCBuiltin<"__builtin_ia32_getmantps512_mask">,
3444           Intrinsic<[llvm_v16f32_ty],
3445           [llvm_v16f32_ty,llvm_i32_ty, llvm_v16f32_ty,llvm_i16_ty,llvm_i32_ty],
3446           [IntrNoMem, ImmArg<1>, ImmArg<4>]>;
3447
3448   def int_x86_avx512_mask_getmant_ss :
3449          GCCBuiltin<"__builtin_ia32_getmantss_round_mask">,
3450           Intrinsic<[llvm_v4f32_ty],
3451           [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty,
3452            llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>, ImmArg<5>]>;
3453
3454   def int_x86_avx512_mask_getmant_sd :
3455          GCCBuiltin<"__builtin_ia32_getmantsd_round_mask">,
3456           Intrinsic<[llvm_v2f64_ty],
3457           [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty,
3458            llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>, ImmArg<5>]>;
3459
3460   def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">,
3461         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3462                                     llvm_i8_ty], [IntrNoMem]>;
3463   def int_x86_avx512_rsqrt14_sd : GCCBuiltin<"__builtin_ia32_rsqrt14sd_mask">,
3464         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3465                                     llvm_i8_ty], [IntrNoMem]>;
3466
3467   def int_x86_avx512_rsqrt14_pd_128 : GCCBuiltin<"__builtin_ia32_rsqrt14pd128_mask">,
3468         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3469                                     llvm_i8_ty], [IntrNoMem]>;
3470   def int_x86_avx512_rsqrt14_pd_256 : GCCBuiltin<"__builtin_ia32_rsqrt14pd256_mask">,
3471         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
3472                                     llvm_i8_ty], [IntrNoMem]>;
3473   def int_x86_avx512_rsqrt14_pd_512 : GCCBuiltin<"__builtin_ia32_rsqrt14pd512_mask">,
3474         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3475                                     llvm_i8_ty], [IntrNoMem]>;
3476   def int_x86_avx512_rsqrt14_ps_128 : GCCBuiltin<"__builtin_ia32_rsqrt14ps128_mask">,
3477         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3478                                      llvm_i8_ty], [IntrNoMem]>;
3479   def int_x86_avx512_rsqrt14_ps_256 : GCCBuiltin<"__builtin_ia32_rsqrt14ps256_mask">,
3480           Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
3481                                      llvm_i8_ty], [IntrNoMem]>;
3482   def int_x86_avx512_rsqrt14_ps_512 : GCCBuiltin<"__builtin_ia32_rsqrt14ps512_mask">,
3483         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3484                                      llvm_i16_ty], [IntrNoMem]>;
3485   def int_x86_avx512_rcp14_ss : GCCBuiltin<"__builtin_ia32_rcp14ss_mask">,
3486         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
3487                                     llvm_i8_ty], [IntrNoMem]>;
3488   def int_x86_avx512_rcp14_sd : GCCBuiltin<"__builtin_ia32_rcp14sd_mask">,
3489         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
3490                                     llvm_i8_ty], [IntrNoMem]>;
3491
3492   def int_x86_avx512_rcp14_pd_128 : GCCBuiltin<"__builtin_ia32_rcp14pd128_mask">,
3493         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3494                                     llvm_i8_ty], [IntrNoMem]>;
3495   def int_x86_avx512_rcp14_pd_256 : GCCBuiltin<"__builtin_ia32_rcp14pd256_mask">,
3496         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
3497                                     llvm_i8_ty], [IntrNoMem]>;
3498   def int_x86_avx512_rcp14_pd_512 : GCCBuiltin<"__builtin_ia32_rcp14pd512_mask">,
3499         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3500                                     llvm_i8_ty], [IntrNoMem]>;
3501   def int_x86_avx512_rcp14_ps_128 : GCCBuiltin<"__builtin_ia32_rcp14ps128_mask">,
3502         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3503                                      llvm_i8_ty], [IntrNoMem]>;
3504   def int_x86_avx512_rcp14_ps_256 : GCCBuiltin<"__builtin_ia32_rcp14ps256_mask">,
3505           Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
3506                                      llvm_i8_ty], [IntrNoMem]>;
3507   def int_x86_avx512_rcp14_ps_512 : GCCBuiltin<"__builtin_ia32_rcp14ps512_mask">,
3508         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3509                                      llvm_i16_ty], [IntrNoMem]>;
3510
3511   def int_x86_avx512_rcp28_ps : GCCBuiltin<"__builtin_ia32_rcp28ps_mask">,
3512             Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3513                                          llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>;
3514   def int_x86_avx512_rcp28_pd : GCCBuiltin<"__builtin_ia32_rcp28pd_mask">,
3515             Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3516                                         llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>;
3517   def int_x86_avx512_exp2_ps : GCCBuiltin<"__builtin_ia32_exp2ps_mask">,
3518             Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3519                                          llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>;
3520   def int_x86_avx512_exp2_pd : GCCBuiltin<"__builtin_ia32_exp2pd_mask">,
3521             Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3522                                         llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>;
3523
3524   def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_round_mask">,
3525             Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3526                                         llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty],
3527                       [IntrNoMem, ImmArg<4>]>;
3528   def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_round_mask">,
3529             Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3530                                         llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty],
3531                       [IntrNoMem, ImmArg<4>]>;
3532   def int_x86_avx512_rsqrt28_ps : GCCBuiltin<"__builtin_ia32_rsqrt28ps_mask">,
3533             Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
3534                                          llvm_i16_ty, llvm_i32_ty],
3535                       [IntrNoMem, ImmArg<3>]>;
3536   def int_x86_avx512_rsqrt28_pd : GCCBuiltin<"__builtin_ia32_rsqrt28pd_mask">,
3537             Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
3538                                         llvm_i8_ty, llvm_i32_ty],
3539                       [IntrNoMem, ImmArg<3>]>;
3540   def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_round_mask">,
3541             Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
3542                                         llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty],
3543                       [IntrNoMem, ImmArg<4>]>;
3544   def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_round_mask">,
3545             Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
3546                                         llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty],
3547                       [IntrNoMem, ImmArg<4>]>;
3548   def int_x86_avx512_psad_bw_512 : GCCBuiltin<"__builtin_ia32_psadbw512">,
3549               Intrinsic<[llvm_v8i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty],
3550                         [IntrNoMem, Commutative]>;
3551 }
3552 // Integer arithmetic ops
3553 let TargetPrefix = "x86" in {
3554   def int_x86_avx512_pmulhu_w_512 : GCCBuiltin<"__builtin_ia32_pmulhuw512">,
3555               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
3556                          llvm_v32i16_ty], [IntrNoMem, Commutative]>;
3557   def int_x86_avx512_pmulh_w_512 : GCCBuiltin<"__builtin_ia32_pmulhw512">,
3558               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
3559                          llvm_v32i16_ty], [IntrNoMem, Commutative]>;
3560   def int_x86_avx512_pavg_b_512 : GCCBuiltin<"__builtin_ia32_pavgb512">,
3561           Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty],
3562                     [IntrNoMem]>;
3563   def int_x86_avx512_pavg_w_512 : GCCBuiltin<"__builtin_ia32_pavgw512">,
3564           Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
3565                     [IntrNoMem]>;
3566   def int_x86_avx512_pmaddw_d_512 : GCCBuiltin<"__builtin_ia32_pmaddwd512">,
3567               Intrinsic<[llvm_v16i32_ty], [llvm_v32i16_ty,
3568                          llvm_v32i16_ty], [IntrNoMem, Commutative]>;
3569   def int_x86_avx512_pmaddubs_w_512 : GCCBuiltin<"__builtin_ia32_pmaddubsw512">,
3570               Intrinsic<[llvm_v32i16_ty], [llvm_v64i8_ty,
3571                          llvm_v64i8_ty], [IntrNoMem]>;
3572
3573   def int_x86_avx512_dbpsadbw_128 :
3574          GCCBuiltin<"__builtin_ia32_dbpsadbw128">,
3575           Intrinsic<[llvm_v8i16_ty],
3576                     [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty],
3577                     [IntrNoMem, ImmArg<2>]>;
3578
3579   def int_x86_avx512_dbpsadbw_256 :
3580          GCCBuiltin<"__builtin_ia32_dbpsadbw256">,
3581           Intrinsic<[llvm_v16i16_ty],
3582                     [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty],
3583                     [IntrNoMem, ImmArg<2>]>;
3584
3585   def int_x86_avx512_dbpsadbw_512 :
3586          GCCBuiltin<"__builtin_ia32_dbpsadbw512">,
3587           Intrinsic<[llvm_v32i16_ty],
3588                     [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty],
3589                     [IntrNoMem, ImmArg<2>]>;
3590 }
3591
3592 // Gather and Scatter ops
3593 let TargetPrefix = "x86" in {
3594   // NOTE: These are deprecated in favor of the versions that take a vXi1 mask.
3595   // NOTE: These can't be ArgMemOnly because you can put the address completely
3596   // in the index register.
3597   def int_x86_avx512_gather_dpd_512  :
3598           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
3599                      llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
3600                     [IntrReadMem, ImmArg<4>]>;
3601   def int_x86_avx512_gather_dps_512  :
3602           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty,
3603                      llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty],
3604                     [IntrReadMem, ImmArg<4>]>;
3605   def int_x86_avx512_gather_qpd_512  :
3606           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
3607                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
3608                     [IntrReadMem, ImmArg<4>]>;
3609   def int_x86_avx512_gather_qps_512  :
3610           Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty,
3611                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
3612                     [IntrReadMem, ImmArg<4>]>;
3613
3614
3615   def int_x86_avx512_gather_dpq_512  :
3616           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
3617                      llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
3618                     [IntrReadMem, ImmArg<4>]>;
3619   def int_x86_avx512_gather_dpi_512  :
3620           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty,
3621                      llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty],
3622                     [IntrReadMem, ImmArg<4>]>;
3623   def int_x86_avx512_gather_qpq_512  :
3624           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
3625                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
3626                     [IntrReadMem, ImmArg<4>]>;
3627   def int_x86_avx512_gather_qpi_512  :
3628           Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty,
3629                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
3630                     [IntrReadMem, ImmArg<4>]>;
3631
3632   def int_x86_avx512_gather3div2_df :
3633           Intrinsic<[llvm_v2f64_ty],
3634           [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
3635           [IntrReadMem, ImmArg<4>]>;
3636
3637   def int_x86_avx512_gather3div2_di :
3638           Intrinsic<[llvm_v2i64_ty],
3639           [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
3640           [IntrReadMem, ImmArg<4>]>;
3641
3642   def int_x86_avx512_gather3div4_df :
3643           Intrinsic<[llvm_v4f64_ty],
3644           [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
3645           [IntrReadMem, ImmArg<4>]>;
3646
3647   def int_x86_avx512_gather3div4_di :
3648           Intrinsic<[llvm_v4i64_ty],
3649           [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
3650           [IntrReadMem, ImmArg<4>]>;
3651
3652   def int_x86_avx512_gather3div4_sf :
3653           Intrinsic<[llvm_v4f32_ty],
3654           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
3655           [IntrReadMem, ImmArg<4>]>;
3656
3657   def int_x86_avx512_gather3div4_si :
3658           Intrinsic<[llvm_v4i32_ty],
3659           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
3660           [IntrReadMem, ImmArg<4>]>;
3661
3662   def int_x86_avx512_gather3div8_sf :
3663           Intrinsic<[llvm_v4f32_ty],
3664           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
3665           [IntrReadMem, ImmArg<4>]>;
3666
3667   def int_x86_avx512_gather3div8_si :
3668           Intrinsic<[llvm_v4i32_ty],
3669           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
3670           [IntrReadMem, ImmArg<4>]>;
3671
3672   def int_x86_avx512_gather3siv2_df :
3673           Intrinsic<[llvm_v2f64_ty],
3674           [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
3675           [IntrReadMem, ImmArg<4>]>;
3676
3677   def int_x86_avx512_gather3siv2_di :
3678           Intrinsic<[llvm_v2i64_ty],
3679           [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
3680           [IntrReadMem, ImmArg<4>]>;
3681
3682   def int_x86_avx512_gather3siv4_df :
3683           Intrinsic<[llvm_v4f64_ty],
3684           [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
3685           [IntrReadMem, ImmArg<4>]>;
3686
3687   def int_x86_avx512_gather3siv4_di :
3688           Intrinsic<[llvm_v4i64_ty],
3689           [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
3690           [IntrReadMem, ImmArg<4>]>;
3691
3692   def int_x86_avx512_gather3siv4_sf :
3693           Intrinsic<[llvm_v4f32_ty],
3694           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
3695           [IntrReadMem, ImmArg<4>]>;
3696
3697   def int_x86_avx512_gather3siv4_si :
3698           Intrinsic<[llvm_v4i32_ty],
3699           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
3700           [IntrReadMem, ImmArg<4>]>;
3701
3702   def int_x86_avx512_gather3siv8_sf :
3703           Intrinsic<[llvm_v8f32_ty],
3704           [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
3705           [IntrReadMem, ImmArg<4>]>;
3706
3707   def int_x86_avx512_gather3siv8_si :
3708           Intrinsic<[llvm_v8i32_ty],
3709           [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
3710           [IntrReadMem, ImmArg<4>]>;
3711
3712 // scatter
3713   // NOTE: These are deprecated in favor of the versions that take a vXi1 mask.
3714   // NOTE: These can't be ArgMemOnly because you can put the address completely
3715   // in the index register.
3716   def int_x86_avx512_scatter_dpd_512  :
3717           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
3718                         llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty],
3719                     [ImmArg<4>]>;
3720   def int_x86_avx512_scatter_dps_512  :
3721           Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty,
3722                        llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty],
3723                     [ImmArg<4>]>;
3724   def int_x86_avx512_scatter_qpd_512  :
3725           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
3726                      llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty],
3727                     [ImmArg<4>]>;
3728   def int_x86_avx512_scatter_qps_512  :
3729           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
3730                      llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty],
3731                     [ImmArg<4>]>;
3732
3733
3734   def int_x86_avx512_scatter_dpq_512  :
3735           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
3736                          llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty],
3737                     [ImmArg<4>]>;
3738   def int_x86_avx512_scatter_dpi_512  :
3739           Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty,
3740                      llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty],
3741                     [ImmArg<4>]>;
3742   def int_x86_avx512_scatter_qpq_512  :
3743           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,llvm_v8i64_ty, llvm_v8i64_ty,
3744                          llvm_i32_ty],
3745                     [ImmArg<4>]>;
3746   def int_x86_avx512_scatter_qpi_512  :
3747           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8i32_ty,
3748                          llvm_i32_ty],
3749                     [ImmArg<4>]>;
3750
3751   def int_x86_avx512_scatterdiv2_df :
3752         Intrinsic<[],
3753         [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty],
3754         [ImmArg<4>]>;
3755
3756   def int_x86_avx512_scatterdiv2_di :
3757           Intrinsic<[],
3758           [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
3759           [ImmArg<4>]>;
3760
3761   def int_x86_avx512_scatterdiv4_df :
3762           Intrinsic<[],
3763           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty],
3764           [ImmArg<4>]>;
3765
3766   def int_x86_avx512_scatterdiv4_di :
3767           Intrinsic<[],
3768           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty],
3769           [ImmArg<4>]>;
3770
3771   def int_x86_avx512_scatterdiv4_sf :
3772           Intrinsic<[],
3773           [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty],
3774           [ImmArg<4>]>;
3775
3776   def int_x86_avx512_scatterdiv4_si :
3777           Intrinsic<[],
3778           [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty],
3779           [ImmArg<4>]>;
3780
3781   def int_x86_avx512_scatterdiv8_sf :
3782           Intrinsic<[],
3783           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty],
3784           [ImmArg<4>]>;
3785
3786   def int_x86_avx512_scatterdiv8_si :
3787           Intrinsic<[],
3788           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty],
3789           [ImmArg<4>]>;
3790
3791   def int_x86_avx512_scattersiv2_df :
3792           Intrinsic<[],
3793           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty],
3794           [ImmArg<4>]>;
3795
3796   def int_x86_avx512_scattersiv2_di :
3797           Intrinsic<[],
3798           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty],
3799           [ImmArg<4>]>;
3800
3801   def int_x86_avx512_scattersiv4_df :
3802           Intrinsic<[],
3803           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty],
3804           [ImmArg<4>]>;
3805
3806   def int_x86_avx512_scattersiv4_di :
3807           Intrinsic<[],
3808           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty],
3809           [ImmArg<4>]>;
3810
3811   def int_x86_avx512_scattersiv4_sf :
3812           Intrinsic<[],
3813           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty],
3814           [ImmArg<4>]>;
3815
3816   def int_x86_avx512_scattersiv4_si :
3817           Intrinsic<[],
3818           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty],
3819           [ImmArg<4>]>;
3820
3821   def int_x86_avx512_scattersiv8_sf :
3822           Intrinsic<[],
3823           [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty],
3824           [ImmArg<4>]>;
3825
3826   def int_x86_avx512_scattersiv8_si :
3827           Intrinsic<[],
3828           [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty],
3829           [ImmArg<4>]>;
3830
3831   // gather prefetch
3832   // NOTE: These can't be ArgMemOnly because you can put the address completely
3833   // in the index register.
3834   def int_x86_avx512_gatherpf_dpd_512  : GCCBuiltin<"__builtin_ia32_gatherpfdpd">,
3835           Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty,
3836                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3837   def int_x86_avx512_gatherpf_dps_512  : GCCBuiltin<"__builtin_ia32_gatherpfdps">,
3838           Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty,
3839                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3840   def int_x86_avx512_gatherpf_qpd_512  : GCCBuiltin<"__builtin_ia32_gatherpfqpd">,
3841           Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty,
3842                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3843   def int_x86_avx512_gatherpf_qps_512  : GCCBuiltin<"__builtin_ia32_gatherpfqps">,
3844           Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty,
3845                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3846
3847   // scatter prefetch
3848   // NOTE: These can't be ArgMemOnly because you can put the address completely
3849   // in the index register.
3850   def int_x86_avx512_scatterpf_dpd_512  : GCCBuiltin<"__builtin_ia32_scatterpfdpd">,
3851           Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty,
3852                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3853   def int_x86_avx512_scatterpf_dps_512  : GCCBuiltin<"__builtin_ia32_scatterpfdps">,
3854           Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty,
3855                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3856   def int_x86_avx512_scatterpf_qpd_512  : GCCBuiltin<"__builtin_ia32_scatterpfqpd">,
3857           Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty,
3858                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3859   def int_x86_avx512_scatterpf_qps_512  : GCCBuiltin<"__builtin_ia32_scatterpfqps">,
3860           Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty,
3861                      llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>;
3862 }
3863
3864 // AVX512 gather/scatter intrinsics that use vXi1 masks.
3865 let TargetPrefix = "x86" in {
3866   // NOTE: These can't be ArgMemOnly because you can put the address completely
3867   // in the index register.
3868   def int_x86_avx512_mask_gather_dpd_512  :
3869           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
3870                      llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
3871                     [IntrReadMem, ImmArg<4>]>;
3872   def int_x86_avx512_mask_gather_dps_512  :
3873           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty,
3874                      llvm_v16i32_ty, llvm_v16i1_ty, llvm_i32_ty],
3875                     [IntrReadMem, ImmArg<4>]>;
3876   def int_x86_avx512_mask_gather_qpd_512  :
3877           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
3878                      llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
3879                     [IntrReadMem, ImmArg<4>]>;
3880   def int_x86_avx512_mask_gather_qps_512  :
3881           Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty,
3882                      llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
3883                     [IntrReadMem, ImmArg<4>]>;
3884
3885
3886   def int_x86_avx512_mask_gather_dpq_512  :
3887           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
3888                      llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
3889                     [IntrReadMem, ImmArg<4>]>;
3890   def int_x86_avx512_mask_gather_dpi_512  :
3891           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty,
3892                      llvm_v16i32_ty, llvm_v16i1_ty, llvm_i32_ty],
3893                     [IntrReadMem, ImmArg<4>]>;
3894   def int_x86_avx512_mask_gather_qpq_512  :
3895           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
3896                      llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
3897                     [IntrReadMem, ImmArg<4>]>;
3898   def int_x86_avx512_mask_gather_qpi_512  :
3899           Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty,
3900                      llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
3901                     [IntrReadMem, ImmArg<4>]>;
3902
3903   def int_x86_avx512_mask_gather3div2_df :
3904           Intrinsic<[llvm_v2f64_ty],
3905           [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
3906           [IntrReadMem, ImmArg<4>]>;
3907
3908   def int_x86_avx512_mask_gather3div2_di :
3909           Intrinsic<[llvm_v2i64_ty],
3910           [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
3911           [IntrReadMem, ImmArg<4>]>;
3912
3913   def int_x86_avx512_mask_gather3div4_df :
3914           Intrinsic<[llvm_v4f64_ty],
3915           [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
3916           [IntrReadMem, ImmArg<4>]>;
3917
3918   def int_x86_avx512_mask_gather3div4_di :
3919           Intrinsic<[llvm_v4i64_ty],
3920           [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
3921           [IntrReadMem, ImmArg<4>]>;
3922
3923   def int_x86_avx512_mask_gather3div4_sf :
3924           Intrinsic<[llvm_v4f32_ty],
3925           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
3926           [IntrReadMem, ImmArg<4>]>;
3927
3928   def int_x86_avx512_mask_gather3div4_si :
3929           Intrinsic<[llvm_v4i32_ty],
3930           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
3931           [IntrReadMem, ImmArg<4>]>;
3932
3933   def int_x86_avx512_mask_gather3div8_sf :
3934           Intrinsic<[llvm_v4f32_ty],
3935           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
3936           [IntrReadMem, ImmArg<4>]>;
3937
3938   def int_x86_avx512_mask_gather3div8_si :
3939           Intrinsic<[llvm_v4i32_ty],
3940           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
3941           [IntrReadMem, ImmArg<4>]>;
3942
3943   def int_x86_avx512_mask_gather3siv2_df :
3944           Intrinsic<[llvm_v2f64_ty],
3945           [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i1_ty, llvm_i32_ty],
3946           [IntrReadMem, ImmArg<4>]>;
3947
3948   def int_x86_avx512_mask_gather3siv2_di :
3949           Intrinsic<[llvm_v2i64_ty],
3950           [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i1_ty, llvm_i32_ty],
3951           [IntrReadMem, ImmArg<4>]>;
3952
3953   def int_x86_avx512_mask_gather3siv4_df :
3954           Intrinsic<[llvm_v4f64_ty],
3955           [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
3956           [IntrReadMem, ImmArg<4>]>;
3957
3958   def int_x86_avx512_mask_gather3siv4_di :
3959           Intrinsic<[llvm_v4i64_ty],
3960           [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
3961           [IntrReadMem, ImmArg<4>]>;
3962
3963   def int_x86_avx512_mask_gather3siv4_sf :
3964           Intrinsic<[llvm_v4f32_ty],
3965           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
3966           [IntrReadMem, ImmArg<4>]>;
3967
3968   def int_x86_avx512_mask_gather3siv4_si :
3969           Intrinsic<[llvm_v4i32_ty],
3970           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
3971           [IntrReadMem, ImmArg<4>]>;
3972
3973   def int_x86_avx512_mask_gather3siv8_sf :
3974           Intrinsic<[llvm_v8f32_ty],
3975           [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
3976           [IntrReadMem, ImmArg<4>]>;
3977
3978   def int_x86_avx512_mask_gather3siv8_si :
3979           Intrinsic<[llvm_v8i32_ty],
3980           [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
3981           [IntrReadMem, ImmArg<4>]>;
3982
3983   def int_x86_avx512_mask_scatter_dpd_512  :
3984           Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
3985                         llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty],
3986                     [ImmArg<4>]>;
3987   def int_x86_avx512_mask_scatter_dps_512  :
3988           Intrinsic<[], [llvm_ptr_ty, llvm_v16i1_ty,
3989                        llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty],
3990                     [ImmArg<4>]>;
3991   def int_x86_avx512_mask_scatter_qpd_512  :
3992           Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
3993                      llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty],
3994                     [ImmArg<4>]>;
3995   def int_x86_avx512_mask_scatter_qps_512  :
3996           Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
3997                      llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty],
3998                     [ImmArg<4>]>;
3999
4000
4001   // NOTE: These can't be ArgMemOnly because you can put the address completely
4002   // in the index register.
4003   def int_x86_avx512_mask_scatter_dpq_512  :
4004           Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
4005                          llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty],
4006                     [ImmArg<4>]>;
4007   def int_x86_avx512_mask_scatter_dpi_512  :
4008           Intrinsic<[], [llvm_ptr_ty, llvm_v16i1_ty,
4009                      llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty],
4010                     [ImmArg<4>]>;
4011   def int_x86_avx512_mask_scatter_qpq_512  :
4012           Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,llvm_v8i64_ty, llvm_v8i64_ty,
4013                          llvm_i32_ty],
4014                     [ImmArg<4>]>;
4015   def int_x86_avx512_mask_scatter_qpi_512  :
4016           Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i64_ty, llvm_v8i32_ty,
4017                          llvm_i32_ty],
4018                     [ImmArg<4>]>;
4019
4020   def int_x86_avx512_mask_scatterdiv2_df :
4021         Intrinsic<[],
4022         [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty],
4023         [ImmArg<4>]>;
4024
4025   def int_x86_avx512_mask_scatterdiv2_di :
4026           Intrinsic<[],
4027           [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
4028           [ImmArg<4>]>;
4029
4030   def int_x86_avx512_mask_scatterdiv4_df :
4031           Intrinsic<[],
4032           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty],
4033           [ImmArg<4>]>;
4034
4035   def int_x86_avx512_mask_scatterdiv4_di :
4036           Intrinsic<[],
4037           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty],
4038           [ImmArg<4>]>;
4039
4040   def int_x86_avx512_mask_scatterdiv4_sf :
4041           Intrinsic<[],
4042           [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty],
4043           [ImmArg<4>]>;
4044
4045   def int_x86_avx512_mask_scatterdiv4_si :
4046           Intrinsic<[],
4047           [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty],
4048           [ImmArg<4>]>;
4049
4050   def int_x86_avx512_mask_scatterdiv8_sf :
4051           Intrinsic<[],
4052           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty],
4053           [ImmArg<4>]>;
4054
4055   def int_x86_avx512_mask_scatterdiv8_si :
4056           Intrinsic<[],
4057           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty],
4058           [ImmArg<4>]>;
4059
4060   def int_x86_avx512_mask_scattersiv2_df :
4061           Intrinsic<[],
4062           [llvm_ptr_ty, llvm_v2i1_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty],
4063           [ImmArg<4>]>;
4064
4065   def int_x86_avx512_mask_scattersiv2_di :
4066           Intrinsic<[],
4067           [llvm_ptr_ty, llvm_v2i1_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty],
4068           [ImmArg<4>]>;
4069
4070   def int_x86_avx512_mask_scattersiv4_df :
4071           Intrinsic<[],
4072           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty],
4073           [ImmArg<4>]>;
4074
4075   def int_x86_avx512_mask_scattersiv4_di :
4076           Intrinsic<[],
4077           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty],
4078           [ImmArg<4>]>;
4079
4080   def int_x86_avx512_mask_scattersiv4_sf :
4081           Intrinsic<[],
4082           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty],
4083           [ImmArg<4>]>;
4084
4085   def int_x86_avx512_mask_scattersiv4_si :
4086           Intrinsic<[],
4087           [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty],
4088           [ImmArg<4>]>;
4089
4090   def int_x86_avx512_mask_scattersiv8_sf :
4091           Intrinsic<[],
4092           [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty],
4093           [ImmArg<4>]>;
4094
4095   def int_x86_avx512_mask_scattersiv8_si :
4096           Intrinsic<[],
4097           [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty],
4098           [ImmArg<4>]>;
4099 }
4100
4101 // AVX-512 conflict detection instruction
4102 // Instructions that count the number of leading zero bits
4103 let TargetPrefix = "x86" in {
4104   def int_x86_avx512_conflict_d_128 :
4105           GCCBuiltin<"__builtin_ia32_vpconflictsi_128">,
4106           Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>;
4107   def int_x86_avx512_conflict_d_256 :
4108           GCCBuiltin<"__builtin_ia32_vpconflictsi_256">,
4109           Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>;
4110   def int_x86_avx512_conflict_d_512 :
4111           GCCBuiltin<"__builtin_ia32_vpconflictsi_512">,
4112           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty], [IntrNoMem]>;
4113
4114   def int_x86_avx512_conflict_q_128 :
4115           GCCBuiltin<"__builtin_ia32_vpconflictdi_128">,
4116           Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>;
4117   def int_x86_avx512_conflict_q_256 :
4118           GCCBuiltin<"__builtin_ia32_vpconflictdi_256">,
4119           Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty], [IntrNoMem]>;
4120   def int_x86_avx512_conflict_q_512 :
4121           GCCBuiltin<"__builtin_ia32_vpconflictdi_512">,
4122           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty], [IntrNoMem]>;
4123 }
4124
4125 // Compares
4126 let TargetPrefix = "x86" in {
4127   // 512-bit
4128   def int_x86_avx512_vcomi_sd : GCCBuiltin<"__builtin_ia32_vcomisd">,
4129               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty,
4130                          llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty],
4131                         [IntrNoMem, ImmArg<2>, ImmArg<3>]>;
4132   def int_x86_avx512_vcomi_ss : GCCBuiltin<"__builtin_ia32_vcomiss">,
4133               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty,
4134                          llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty],
4135                         [IntrNoMem, ImmArg<2>, ImmArg<3>]>;
4136 }
4137
4138 // Compress, Expand
4139 let TargetPrefix = "x86" in {
4140   def int_x86_avx512_mask_compress :
4141         Intrinsic<[llvm_anyvector_ty],
4142                   [LLVMMatchType<0>, LLVMMatchType<0>,
4143                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
4144                   [IntrNoMem]>;
4145   def int_x86_avx512_mask_expand :
4146         Intrinsic<[llvm_anyvector_ty],
4147                   [LLVMMatchType<0>, LLVMMatchType<0>,
4148                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
4149                   [IntrNoMem]>;
4150 }
4151
4152 // truncate
4153 let TargetPrefix = "x86" in {
4154   def int_x86_avx512_mask_pmov_qb_128 :
4155           GCCBuiltin<"__builtin_ia32_pmovqb128_mask">,
4156           Intrinsic<[llvm_v16i8_ty],
4157                     [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4158                     [IntrNoMem]>;
4159   def int_x86_avx512_mask_pmov_qb_mem_128 :
4160           GCCBuiltin<"__builtin_ia32_pmovqb128mem_mask">,
4161           Intrinsic<[],
4162                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4163                     [IntrArgMemOnly]>;
4164   def int_x86_avx512_mask_pmovs_qb_128 :
4165           GCCBuiltin<"__builtin_ia32_pmovsqb128_mask">,
4166           Intrinsic<[llvm_v16i8_ty],
4167                     [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4168                     [IntrNoMem]>;
4169   def int_x86_avx512_mask_pmovs_qb_mem_128 :
4170           GCCBuiltin<"__builtin_ia32_pmovsqb128mem_mask">,
4171           Intrinsic<[],
4172                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4173                     [IntrArgMemOnly]>;
4174   def int_x86_avx512_mask_pmovus_qb_128 :
4175           GCCBuiltin<"__builtin_ia32_pmovusqb128_mask">,
4176           Intrinsic<[llvm_v16i8_ty],
4177                     [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4178                     [IntrNoMem]>;
4179   def int_x86_avx512_mask_pmovus_qb_mem_128 :
4180           GCCBuiltin<"__builtin_ia32_pmovusqb128mem_mask">,
4181           Intrinsic<[],
4182                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4183                     [IntrArgMemOnly]>;
4184   def int_x86_avx512_mask_pmov_qb_256 :
4185           GCCBuiltin<"__builtin_ia32_pmovqb256_mask">,
4186           Intrinsic<[llvm_v16i8_ty],
4187                     [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4188                     [IntrNoMem]>;
4189   def int_x86_avx512_mask_pmov_qb_mem_256 :
4190           GCCBuiltin<"__builtin_ia32_pmovqb256mem_mask">,
4191           Intrinsic<[],
4192                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4193                     [IntrArgMemOnly]>;
4194   def int_x86_avx512_mask_pmovs_qb_256 :
4195           GCCBuiltin<"__builtin_ia32_pmovsqb256_mask">,
4196           Intrinsic<[llvm_v16i8_ty],
4197                     [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4198                     [IntrNoMem]>;
4199   def int_x86_avx512_mask_pmovs_qb_mem_256 :
4200           GCCBuiltin<"__builtin_ia32_pmovsqb256mem_mask">,
4201           Intrinsic<[],
4202                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4203                     [IntrArgMemOnly]>;
4204   def int_x86_avx512_mask_pmovus_qb_256 :
4205           GCCBuiltin<"__builtin_ia32_pmovusqb256_mask">,
4206           Intrinsic<[llvm_v16i8_ty],
4207                     [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4208                     [IntrNoMem]>;
4209   def int_x86_avx512_mask_pmovus_qb_mem_256 :
4210           GCCBuiltin<"__builtin_ia32_pmovusqb256mem_mask">,
4211           Intrinsic<[],
4212                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4213                     [IntrArgMemOnly]>;
4214   def int_x86_avx512_mask_pmov_qb_512 :
4215           GCCBuiltin<"__builtin_ia32_pmovqb512_mask">,
4216           Intrinsic<[llvm_v16i8_ty],
4217                     [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4218                     [IntrNoMem]>;
4219   def int_x86_avx512_mask_pmov_qb_mem_512 :
4220           GCCBuiltin<"__builtin_ia32_pmovqb512mem_mask">,
4221           Intrinsic<[],
4222                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4223                     [IntrArgMemOnly]>;
4224   def int_x86_avx512_mask_pmovs_qb_512 :
4225           GCCBuiltin<"__builtin_ia32_pmovsqb512_mask">,
4226           Intrinsic<[llvm_v16i8_ty],
4227                     [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4228                     [IntrNoMem]>;
4229   def int_x86_avx512_mask_pmovs_qb_mem_512 :
4230           GCCBuiltin<"__builtin_ia32_pmovsqb512mem_mask">,
4231           Intrinsic<[],
4232                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4233                     [IntrArgMemOnly]>;
4234   def int_x86_avx512_mask_pmovus_qb_512 :
4235           GCCBuiltin<"__builtin_ia32_pmovusqb512_mask">,
4236           Intrinsic<[llvm_v16i8_ty],
4237                     [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty],
4238                     [IntrNoMem]>;
4239   def int_x86_avx512_mask_pmovus_qb_mem_512 :
4240           GCCBuiltin<"__builtin_ia32_pmovusqb512mem_mask">,
4241           Intrinsic<[],
4242                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4243                     [IntrArgMemOnly]>;
4244   def int_x86_avx512_mask_pmov_qw_128 :
4245           GCCBuiltin<"__builtin_ia32_pmovqw128_mask">,
4246           Intrinsic<[llvm_v8i16_ty],
4247                     [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4248                     [IntrNoMem]>;
4249   def int_x86_avx512_mask_pmov_qw_mem_128 :
4250           GCCBuiltin<"__builtin_ia32_pmovqw128mem_mask">,
4251           Intrinsic<[],
4252                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4253                     [IntrArgMemOnly]>;
4254   def int_x86_avx512_mask_pmovs_qw_128 :
4255           GCCBuiltin<"__builtin_ia32_pmovsqw128_mask">,
4256           Intrinsic<[llvm_v8i16_ty],
4257                     [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4258                     [IntrNoMem]>;
4259   def int_x86_avx512_mask_pmovs_qw_mem_128 :
4260           GCCBuiltin<"__builtin_ia32_pmovsqw128mem_mask">,
4261           Intrinsic<[],
4262                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4263                     [IntrArgMemOnly]>;
4264   def int_x86_avx512_mask_pmovus_qw_128 :
4265           GCCBuiltin<"__builtin_ia32_pmovusqw128_mask">,
4266           Intrinsic<[llvm_v8i16_ty],
4267                     [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4268                     [IntrNoMem]>;
4269   def int_x86_avx512_mask_pmovus_qw_mem_128 :
4270           GCCBuiltin<"__builtin_ia32_pmovusqw128mem_mask">,
4271           Intrinsic<[],
4272                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4273                     [IntrArgMemOnly]>;
4274   def int_x86_avx512_mask_pmov_qw_256 :
4275           GCCBuiltin<"__builtin_ia32_pmovqw256_mask">,
4276           Intrinsic<[llvm_v8i16_ty],
4277                     [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4278                     [IntrNoMem]>;
4279   def int_x86_avx512_mask_pmov_qw_mem_256 :
4280           GCCBuiltin<"__builtin_ia32_pmovqw256mem_mask">,
4281           Intrinsic<[],
4282                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4283                     [IntrArgMemOnly]>;
4284   def int_x86_avx512_mask_pmovs_qw_256 :
4285           GCCBuiltin<"__builtin_ia32_pmovsqw256_mask">,
4286           Intrinsic<[llvm_v8i16_ty],
4287                     [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4288                     [IntrNoMem]>;
4289   def int_x86_avx512_mask_pmovs_qw_mem_256 :
4290           GCCBuiltin<"__builtin_ia32_pmovsqw256mem_mask">,
4291           Intrinsic<[],
4292                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4293                     [IntrArgMemOnly]>;
4294   def int_x86_avx512_mask_pmovus_qw_256 :
4295           GCCBuiltin<"__builtin_ia32_pmovusqw256_mask">,
4296           Intrinsic<[llvm_v8i16_ty],
4297                     [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4298                     [IntrNoMem]>;
4299   def int_x86_avx512_mask_pmovus_qw_mem_256 :
4300           GCCBuiltin<"__builtin_ia32_pmovusqw256mem_mask">,
4301           Intrinsic<[],
4302                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4303                     [IntrArgMemOnly]>;
4304   def int_x86_avx512_mask_pmov_qw_512 :
4305           Intrinsic<[llvm_v8i16_ty],
4306                     [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4307                     [IntrNoMem]>;
4308   def int_x86_avx512_mask_pmov_qw_mem_512 :
4309           GCCBuiltin<"__builtin_ia32_pmovqw512mem_mask">,
4310           Intrinsic<[],
4311                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4312                     [IntrArgMemOnly]>;
4313   def int_x86_avx512_mask_pmovs_qw_512 :
4314           GCCBuiltin<"__builtin_ia32_pmovsqw512_mask">,
4315           Intrinsic<[llvm_v8i16_ty],
4316                     [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4317                     [IntrNoMem]>;
4318   def int_x86_avx512_mask_pmovs_qw_mem_512 :
4319           GCCBuiltin<"__builtin_ia32_pmovsqw512mem_mask">,
4320           Intrinsic<[],
4321                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4322                     [IntrArgMemOnly]>;
4323   def int_x86_avx512_mask_pmovus_qw_512 :
4324           GCCBuiltin<"__builtin_ia32_pmovusqw512_mask">,
4325           Intrinsic<[llvm_v8i16_ty],
4326                     [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty],
4327                     [IntrNoMem]>;
4328   def int_x86_avx512_mask_pmovus_qw_mem_512 :
4329           GCCBuiltin<"__builtin_ia32_pmovusqw512mem_mask">,
4330           Intrinsic<[],
4331                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4332                     [IntrArgMemOnly]>;
4333   def int_x86_avx512_mask_pmov_qd_128 :
4334           GCCBuiltin<"__builtin_ia32_pmovqd128_mask">,
4335           Intrinsic<[llvm_v4i32_ty],
4336                     [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
4337                     [IntrNoMem]>;
4338   def int_x86_avx512_mask_pmov_qd_mem_128 :
4339           GCCBuiltin<"__builtin_ia32_pmovqd128mem_mask">,
4340           Intrinsic<[],
4341                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4342                     [IntrArgMemOnly]>;
4343   def int_x86_avx512_mask_pmovs_qd_128 :
4344           GCCBuiltin<"__builtin_ia32_pmovsqd128_mask">,
4345           Intrinsic<[llvm_v4i32_ty],
4346                     [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
4347                     [IntrNoMem]>;
4348   def int_x86_avx512_mask_pmovs_qd_mem_128 :
4349           GCCBuiltin<"__builtin_ia32_pmovsqd128mem_mask">,
4350           Intrinsic<[],
4351                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4352                     [IntrArgMemOnly]>;
4353   def int_x86_avx512_mask_pmovus_qd_128 :
4354           GCCBuiltin<"__builtin_ia32_pmovusqd128_mask">,
4355           Intrinsic<[llvm_v4i32_ty],
4356                     [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
4357                     [IntrNoMem]>;
4358   def int_x86_avx512_mask_pmovus_qd_mem_128 :
4359           GCCBuiltin<"__builtin_ia32_pmovusqd128mem_mask">,
4360           Intrinsic<[],
4361                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
4362                     [IntrArgMemOnly]>;
4363   def int_x86_avx512_mask_pmov_qd_mem_256 :
4364           GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">,
4365           Intrinsic<[],
4366                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4367                     [IntrArgMemOnly]>;
4368   def int_x86_avx512_mask_pmovs_qd_256 :
4369           GCCBuiltin<"__builtin_ia32_pmovsqd256_mask">,
4370           Intrinsic<[llvm_v4i32_ty],
4371                     [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
4372                     [IntrNoMem]>;
4373   def int_x86_avx512_mask_pmovs_qd_mem_256 :
4374           GCCBuiltin<"__builtin_ia32_pmovsqd256mem_mask">,
4375           Intrinsic<[],
4376                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4377                     [IntrArgMemOnly]>;
4378   def int_x86_avx512_mask_pmovus_qd_256 :
4379           GCCBuiltin<"__builtin_ia32_pmovusqd256_mask">,
4380           Intrinsic<[llvm_v4i32_ty],
4381                     [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
4382                     [IntrNoMem]>;
4383   def int_x86_avx512_mask_pmovus_qd_mem_256 :
4384           GCCBuiltin<"__builtin_ia32_pmovusqd256mem_mask">,
4385           Intrinsic<[],
4386                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
4387                     [IntrArgMemOnly]>;
4388   def int_x86_avx512_mask_pmov_qd_mem_512 :
4389           GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">,
4390           Intrinsic<[],
4391                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4392                     [IntrArgMemOnly]>;
4393   def int_x86_avx512_mask_pmovs_qd_512 :
4394           GCCBuiltin<"__builtin_ia32_pmovsqd512_mask">,
4395           Intrinsic<[llvm_v8i32_ty],
4396                     [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty],
4397                     [IntrNoMem]>;
4398   def int_x86_avx512_mask_pmovs_qd_mem_512 :
4399           GCCBuiltin<"__builtin_ia32_pmovsqd512mem_mask">,
4400           Intrinsic<[],
4401                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4402                     [IntrArgMemOnly]>;
4403   def int_x86_avx512_mask_pmovus_qd_512 :
4404           GCCBuiltin<"__builtin_ia32_pmovusqd512_mask">,
4405           Intrinsic<[llvm_v8i32_ty],
4406                     [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty],
4407                     [IntrNoMem]>;
4408   def int_x86_avx512_mask_pmovus_qd_mem_512 :
4409           GCCBuiltin<"__builtin_ia32_pmovusqd512mem_mask">,
4410           Intrinsic<[],
4411                     [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
4412                     [IntrArgMemOnly]>;
4413   def int_x86_avx512_mask_pmov_db_128 :
4414           GCCBuiltin<"__builtin_ia32_pmovdb128_mask">,
4415           Intrinsic<[llvm_v16i8_ty],
4416                     [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty],
4417                     [IntrNoMem]>;
4418   def int_x86_avx512_mask_pmov_db_mem_128 :
4419           GCCBuiltin<"__builtin_ia32_pmovdb128mem_mask">,
4420           Intrinsic<[],
4421                     [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
4422                     [IntrArgMemOnly]>;
4423   def int_x86_avx512_mask_pmovs_db_128 :
4424           GCCBuiltin<"__builtin_ia32_pmovsdb128_mask">,
4425           Intrinsic<[llvm_v16i8_ty],
4426                     [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty],
4427                     [IntrNoMem]>;
4428   def int_x86_avx512_mask_pmovs_db_mem_128 :
4429           GCCBuiltin<"__builtin_ia32_pmovsdb128mem_mask">,
4430           Intrinsic<[],
4431                     [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
4432                     [IntrArgMemOnly]>;
4433   def int_x86_avx512_mask_pmovus_db_128 :
4434           GCCBuiltin<"__builtin_ia32_pmovusdb128_mask">,
4435           Intrinsic<[llvm_v16i8_ty],
4436                     [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty],
4437                     [IntrNoMem]>;
4438   def int_x86_avx512_mask_pmovus_db_mem_128 :
4439           GCCBuiltin<"__builtin_ia32_pmovusdb128mem_mask">,
4440           Intrinsic<[],
4441                     [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
4442                     [IntrArgMemOnly]>;
4443   def int_x86_avx512_mask_pmov_db_256 :
4444           GCCBuiltin<"__builtin_ia32_pmovdb256_mask">,
4445           Intrinsic<[llvm_v16i8_ty],
4446                     [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty],
4447                     [IntrNoMem]>;
4448   def int_x86_avx512_mask_pmov_db_mem_256 :
4449           GCCBuiltin<"__builtin_ia32_pmovdb256mem_mask">,
4450           Intrinsic<[],
4451                     [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
4452                     [IntrArgMemOnly]>;
4453   def int_x86_avx512_mask_pmovs_db_256 :
4454           GCCBuiltin<"__builtin_ia32_pmovsdb256_mask">,
4455           Intrinsic<[llvm_v16i8_ty],
4456                     [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty],
4457                     [IntrNoMem]>;
4458   def int_x86_avx512_mask_pmovs_db_mem_256 :
4459           GCCBuiltin<"__builtin_ia32_pmovsdb256mem_mask">,
4460           Intrinsic<[],
4461                     [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
4462                     [IntrArgMemOnly]>;
4463   def int_x86_avx512_mask_pmovus_db_256 :
4464           GCCBuiltin<"__builtin_ia32_pmovusdb256_mask">,
4465           Intrinsic<[llvm_v16i8_ty],
4466                     [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty],
4467                     [IntrNoMem]>;
4468   def int_x86_avx512_mask_pmovus_db_mem_256 :
4469           GCCBuiltin<"__builtin_ia32_pmovusdb256mem_mask">,
4470           Intrinsic<[],
4471                     [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
4472                     [IntrArgMemOnly]>;
4473   def int_x86_avx512_mask_pmov_db_512 :
4474           Intrinsic<[llvm_v16i8_ty],
4475                     [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty],
4476                     [IntrNoMem]>;
4477   def int_x86_avx512_mask_pmov_db_mem_512 :
4478           GCCBuiltin<"__builtin_ia32_pmovdb512mem_mask">,
4479           Intrinsic<[],
4480                     [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
4481                     [IntrArgMemOnly]>;
4482   def int_x86_avx512_mask_pmovs_db_512 :
4483           GCCBuiltin<"__builtin_ia32_pmovsdb512_mask">,
4484           Intrinsic<[llvm_v16i8_ty],
4485                     [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty],
4486                     [IntrNoMem]>;
4487   def int_x86_avx512_mask_pmovs_db_mem_512 :
4488           GCCBuiltin<"__builtin_ia32_pmovsdb512mem_mask">,
4489           Intrinsic<[],
4490                     [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
4491                     [IntrArgMemOnly]>;
4492   def int_x86_avx512_mask_pmovus_db_512 :
4493           GCCBuiltin<"__builtin_ia32_pmovusdb512_mask">,
4494           Intrinsic<[llvm_v16i8_ty],
4495                     [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty],
4496                     [IntrNoMem]>;
4497   def int_x86_avx512_mask_pmovus_db_mem_512 :
4498           GCCBuiltin<"__builtin_ia32_pmovusdb512mem_mask">,
4499           Intrinsic<[],
4500                     [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
4501                     [IntrArgMemOnly]>;
4502   def int_x86_avx512_mask_pmov_dw_128 :
4503           GCCBuiltin<"__builtin_ia32_pmovdw128_mask">,
4504           Intrinsic<[llvm_v8i16_ty],
4505                     [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty],
4506                     [IntrNoMem]>;
4507   def int_x86_avx512_mask_pmov_dw_mem_128 :
4508           GCCBuiltin<"__builtin_ia32_pmovdw128mem_mask">,
4509           Intrinsic<[],
4510                     [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
4511                     [IntrArgMemOnly]>;
4512   def int_x86_avx512_mask_pmovs_dw_128 :
4513           GCCBuiltin<"__builtin_ia32_pmovsdw128_mask">,
4514           Intrinsic<[llvm_v8i16_ty],
4515                     [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty],
4516                     [IntrNoMem]>;
4517   def int_x86_avx512_mask_pmovs_dw_mem_128 :
4518           GCCBuiltin<"__builtin_ia32_pmovsdw128mem_mask">,
4519           Intrinsic<[],
4520                     [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
4521                     [IntrArgMemOnly]>;
4522   def int_x86_avx512_mask_pmovus_dw_128 :
4523           GCCBuiltin<"__builtin_ia32_pmovusdw128_mask">,
4524           Intrinsic<[llvm_v8i16_ty],
4525                     [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty],
4526                     [IntrNoMem]>;
4527   def int_x86_avx512_mask_pmovus_dw_mem_128 :
4528           GCCBuiltin<"__builtin_ia32_pmovusdw128mem_mask">,
4529           Intrinsic<[],
4530                     [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
4531                     [IntrArgMemOnly]>;
4532   def int_x86_avx512_mask_pmov_dw_256 :
4533           GCCBuiltin<"__builtin_ia32_pmovdw256_mask">,
4534           Intrinsic<[llvm_v8i16_ty],
4535                     [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty],
4536                     [IntrNoMem]>;
4537   def int_x86_avx512_mask_pmov_dw_mem_256 :
4538           GCCBuiltin<"__builtin_ia32_pmovdw256mem_mask">,
4539           Intrinsic<[],
4540                     [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
4541                     [IntrArgMemOnly]>;
4542   def int_x86_avx512_mask_pmovs_dw_256 :
4543           GCCBuiltin<"__builtin_ia32_pmovsdw256_mask">,
4544           Intrinsic<[llvm_v8i16_ty],
4545                     [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty],
4546                     [IntrNoMem]>;
4547   def int_x86_avx512_mask_pmovs_dw_mem_256 :
4548           GCCBuiltin<"__builtin_ia32_pmovsdw256mem_mask">,
4549           Intrinsic<[],
4550                     [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
4551                     [IntrArgMemOnly]>;
4552   def int_x86_avx512_mask_pmovus_dw_256 :
4553           GCCBuiltin<"__builtin_ia32_pmovusdw256_mask">,
4554           Intrinsic<[llvm_v8i16_ty],
4555                     [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty],
4556                     [IntrNoMem]>;
4557   def int_x86_avx512_mask_pmovus_dw_mem_256 :
4558           GCCBuiltin<"__builtin_ia32_pmovusdw256mem_mask">,
4559           Intrinsic<[],
4560                     [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
4561                     [IntrArgMemOnly]>;
4562   def int_x86_avx512_mask_pmov_dw_512 :
4563           Intrinsic<[llvm_v16i16_ty],
4564                     [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty],
4565                     [IntrNoMem]>;
4566   def int_x86_avx512_mask_pmov_dw_mem_512 :
4567           GCCBuiltin<"__builtin_ia32_pmovdw512mem_mask">,
4568           Intrinsic<[],
4569                     [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
4570                     [IntrArgMemOnly]>;
4571   def int_x86_avx512_mask_pmovs_dw_512 :
4572           GCCBuiltin<"__builtin_ia32_pmovsdw512_mask">,
4573           Intrinsic<[llvm_v16i16_ty],
4574                     [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty],
4575                     [IntrNoMem]>;
4576   def int_x86_avx512_mask_pmovs_dw_mem_512 :
4577           GCCBuiltin<"__builtin_ia32_pmovsdw512mem_mask">,
4578           Intrinsic<[],
4579                     [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
4580                     [IntrArgMemOnly]>;
4581   def int_x86_avx512_mask_pmovus_dw_512 :
4582           GCCBuiltin<"__builtin_ia32_pmovusdw512_mask">,
4583           Intrinsic<[llvm_v16i16_ty],
4584                     [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty],
4585                     [IntrNoMem]>;
4586   def int_x86_avx512_mask_pmovus_dw_mem_512 :
4587           GCCBuiltin<"__builtin_ia32_pmovusdw512mem_mask">,
4588           Intrinsic<[],
4589                     [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
4590                     [IntrArgMemOnly]>;
4591   def int_x86_avx512_mask_pmov_wb_128 :
4592           GCCBuiltin<"__builtin_ia32_pmovwb128_mask">,
4593           Intrinsic<[llvm_v16i8_ty],
4594                     [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty],
4595                     [IntrNoMem]>;
4596   def int_x86_avx512_mask_pmov_wb_mem_128 :
4597           GCCBuiltin<"__builtin_ia32_pmovwb128mem_mask">,
4598           Intrinsic<[],
4599                     [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
4600                     [IntrArgMemOnly]>;
4601   def int_x86_avx512_mask_pmovs_wb_128 :
4602           GCCBuiltin<"__builtin_ia32_pmovswb128_mask">,
4603           Intrinsic<[llvm_v16i8_ty],
4604                     [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty],
4605                     [IntrNoMem]>;
4606   def int_x86_avx512_mask_pmovs_wb_mem_128 :
4607           GCCBuiltin<"__builtin_ia32_pmovswb128mem_mask">,
4608           Intrinsic<[],
4609                     [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
4610                     [IntrArgMemOnly]>;
4611   def int_x86_avx512_mask_pmovus_wb_128 :
4612           GCCBuiltin<"__builtin_ia32_pmovuswb128_mask">,
4613           Intrinsic<[llvm_v16i8_ty],
4614                     [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty],
4615                     [IntrNoMem]>;
4616   def int_x86_avx512_mask_pmovus_wb_mem_128 :
4617           GCCBuiltin<"__builtin_ia32_pmovuswb128mem_mask">,
4618           Intrinsic<[],
4619                     [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
4620                     [IntrArgMemOnly]>;
4621   def int_x86_avx512_mask_pmov_wb_mem_256 :
4622           GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">,
4623           Intrinsic<[],
4624                     [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
4625                     [IntrArgMemOnly]>;
4626   def int_x86_avx512_mask_pmovs_wb_256 :
4627           GCCBuiltin<"__builtin_ia32_pmovswb256_mask">,
4628           Intrinsic<[llvm_v16i8_ty],
4629                     [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty],
4630                     [IntrNoMem]>;
4631   def int_x86_avx512_mask_pmovs_wb_mem_256 :
4632           GCCBuiltin<"__builtin_ia32_pmovswb256mem_mask">,
4633           Intrinsic<[],
4634                     [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
4635                     [IntrArgMemOnly]>;
4636   def int_x86_avx512_mask_pmovus_wb_256 :
4637           GCCBuiltin<"__builtin_ia32_pmovuswb256_mask">,
4638           Intrinsic<[llvm_v16i8_ty],
4639                     [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty],
4640                     [IntrNoMem]>;
4641   def int_x86_avx512_mask_pmovus_wb_mem_256 :
4642           GCCBuiltin<"__builtin_ia32_pmovuswb256mem_mask">,
4643           Intrinsic<[],
4644                     [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
4645                     [IntrArgMemOnly]>;
4646   def int_x86_avx512_mask_pmov_wb_mem_512 :
4647           GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">,
4648           Intrinsic<[],
4649                     [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty],
4650                     [IntrArgMemOnly]>;
4651   def int_x86_avx512_mask_pmovs_wb_512 :
4652           GCCBuiltin<"__builtin_ia32_pmovswb512_mask">,
4653           Intrinsic<[llvm_v32i8_ty],
4654                     [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty],
4655                     [IntrNoMem]>;
4656   def int_x86_avx512_mask_pmovs_wb_mem_512 :
4657           GCCBuiltin<"__builtin_ia32_pmovswb512mem_mask">,
4658           Intrinsic<[],
4659                     [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty],
4660                     [IntrArgMemOnly]>;
4661   def int_x86_avx512_mask_pmovus_wb_512 :
4662           GCCBuiltin<"__builtin_ia32_pmovuswb512_mask">,
4663           Intrinsic<[llvm_v32i8_ty],
4664                     [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty],
4665                     [IntrNoMem]>;
4666   def int_x86_avx512_mask_pmovus_wb_mem_512 :
4667           GCCBuiltin<"__builtin_ia32_pmovuswb512mem_mask">,
4668           Intrinsic<[],
4669                     [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty],
4670                     [IntrArgMemOnly]>;
4671 }
4672
4673 // Bitwise ternary logic
4674 let TargetPrefix = "x86" in {
4675   def int_x86_avx512_pternlog_d_128 :
4676           GCCBuiltin<"__builtin_ia32_pternlogd128">,
4677           Intrinsic<[llvm_v4i32_ty],
4678                     [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty],
4679                     [IntrNoMem, ImmArg<3>]>;
4680
4681   def int_x86_avx512_pternlog_d_256 :
4682           GCCBuiltin<"__builtin_ia32_pternlogd256">,
4683           Intrinsic<[llvm_v8i32_ty],
4684                     [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty],
4685                     [IntrNoMem, ImmArg<3>]>;
4686
4687   def int_x86_avx512_pternlog_d_512 :
4688           GCCBuiltin<"__builtin_ia32_pternlogd512">,
4689           Intrinsic<[llvm_v16i32_ty],
4690                     [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty,
4691                      llvm_i32_ty], [IntrNoMem, ImmArg<3>]>;
4692
4693   def int_x86_avx512_pternlog_q_128 :
4694           GCCBuiltin<"__builtin_ia32_pternlogq128">,
4695           Intrinsic<[llvm_v2i64_ty],
4696                     [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
4697                     [IntrNoMem, ImmArg<3>]>;
4698
4699   def int_x86_avx512_pternlog_q_256 :
4700           GCCBuiltin<"__builtin_ia32_pternlogq256">,
4701           Intrinsic<[llvm_v4i64_ty],
4702                     [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty],
4703                     [IntrNoMem, ImmArg<3>]>;
4704
4705   def int_x86_avx512_pternlog_q_512 :
4706           GCCBuiltin<"__builtin_ia32_pternlogq512">,
4707           Intrinsic<[llvm_v8i64_ty],
4708                     [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty],
4709                     [IntrNoMem, ImmArg<3>]>;
4710 }
4711
4712 // vp2intersect
4713 let TargetPrefix = "x86" in {
4714   def int_x86_avx512_vp2intersect_q_512 :
4715           Intrinsic<[llvm_v8i1_ty, llvm_v8i1_ty],
4716                     [llvm_v8i64_ty, llvm_v8i64_ty],
4717                     [IntrNoMem]>;
4718   def int_x86_avx512_vp2intersect_q_256 :
4719           Intrinsic<[llvm_v4i1_ty, llvm_v4i1_ty],
4720                     [llvm_v4i64_ty, llvm_v4i64_ty],
4721                     [IntrNoMem]>;
4722   def int_x86_avx512_vp2intersect_q_128 :
4723           Intrinsic<[llvm_v2i1_ty, llvm_v2i1_ty],
4724                     [llvm_v2i64_ty, llvm_v2i64_ty],
4725                     [IntrNoMem]>;
4726   def int_x86_avx512_vp2intersect_d_512 :
4727           Intrinsic<[llvm_v16i1_ty, llvm_v16i1_ty],
4728                     [llvm_v16i32_ty, llvm_v16i32_ty],
4729                     [IntrNoMem]>;
4730   def int_x86_avx512_vp2intersect_d_256 :
4731           Intrinsic<[llvm_v8i1_ty, llvm_v8i1_ty],
4732                     [llvm_v8i32_ty, llvm_v8i32_ty],
4733                     [IntrNoMem]>;
4734   def int_x86_avx512_vp2intersect_d_128 :
4735           Intrinsic<[llvm_v4i1_ty, llvm_v4i1_ty],
4736                     [llvm_v4i32_ty, llvm_v4i32_ty],
4737                     [IntrNoMem]>;
4738 }
4739
4740 // Misc.
4741 let TargetPrefix = "x86" in {
4742   // NOTE: These comparison intrinsics are not used by clang as long as the
4743   //       distinction in signaling behaviour is not implemented.
4744   def int_x86_avx512_cmp_ps_512 :
4745               Intrinsic<[llvm_v16i1_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
4746                          llvm_i32_ty, llvm_i32_ty],
4747                         [IntrNoMem, ImmArg<2>, ImmArg<3>]>;
4748   def int_x86_avx512_cmp_pd_512 :
4749               Intrinsic<[llvm_v8i1_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
4750                          llvm_i32_ty, llvm_i32_ty],
4751                         [IntrNoMem, ImmArg<2>, ImmArg<3>]>;
4752   def int_x86_avx512_cmp_ps_256 :
4753               Intrinsic<[llvm_v8i1_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
4754                          llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
4755   def int_x86_avx512_cmp_pd_256 :
4756               Intrinsic<[llvm_v4i1_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
4757                          llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
4758   def int_x86_avx512_cmp_ps_128 :
4759             Intrinsic<[llvm_v4i1_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
4760                        llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
4761   def int_x86_avx512_cmp_pd_128 :
4762             Intrinsic<[llvm_v2i1_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
4763                        llvm_i32_ty], [IntrNoMem, ImmArg<2>]>;
4764
4765   def int_x86_avx512_mask_cmp_ss :
4766         GCCBuiltin<"__builtin_ia32_cmpss_mask">,
4767               Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
4768                          llvm_i32_ty, llvm_i8_ty, llvm_i32_ty],
4769                         [IntrNoMem, ImmArg<2>, ImmArg<4>]>;
4770   def int_x86_avx512_mask_cmp_sd :
4771         GCCBuiltin<"__builtin_ia32_cmpsd_mask">,
4772               Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
4773                          llvm_i32_ty, llvm_i8_ty, llvm_i32_ty],
4774                         [IntrNoMem, ImmArg<2>, ImmArg<4>]>;
4775 }
4776
4777 //===----------------------------------------------------------------------===//
4778 // SHA intrinsics
4779 let TargetPrefix = "x86" in {
4780   def int_x86_sha1rnds4 : GCCBuiltin<"__builtin_ia32_sha1rnds4">,
4781         Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty],
4782                   [IntrNoMem, ImmArg<2>]>;
4783   def int_x86_sha1nexte : GCCBuiltin<"__builtin_ia32_sha1nexte">,
4784       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
4785   def int_x86_sha1msg1 : GCCBuiltin<"__builtin_ia32_sha1msg1">,
4786       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
4787   def int_x86_sha1msg2 : GCCBuiltin<"__builtin_ia32_sha1msg2">,
4788       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
4789   def int_x86_sha256rnds2 : GCCBuiltin<"__builtin_ia32_sha256rnds2">,
4790       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
4791                 [IntrNoMem]>;
4792   def int_x86_sha256msg1 : GCCBuiltin<"__builtin_ia32_sha256msg1">,
4793       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
4794   def int_x86_sha256msg2 : GCCBuiltin<"__builtin_ia32_sha256msg2">,
4795       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
4796 }
4797
4798 //===----------------------------------------------------------------------===//
4799 // Thread synchronization ops with timer.
4800 let TargetPrefix = "x86" in {
4801   def int_x86_monitorx
4802       : GCCBuiltin<"__builtin_ia32_monitorx">,
4803         Intrinsic<[], [ llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty ], []>;
4804   def int_x86_mwaitx
4805       : GCCBuiltin<"__builtin_ia32_mwaitx">,
4806         Intrinsic<[], [ llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ], []>;
4807 }
4808
4809 //===----------------------------------------------------------------------===//
4810 // Cache-line zero
4811 let TargetPrefix = "x86" in {
4812   def int_x86_clzero : GCCBuiltin<"__builtin_ia32_clzero">,
4813       Intrinsic<[], [llvm_ptr_ty], []>;
4814 }
4815
4816 //===----------------------------------------------------------------------===//
4817 // Cache write back intrinsics
4818
4819 let TargetPrefix = "x86" in {
4820   // Write back and invalidate
4821   def int_x86_wbinvd : GCCBuiltin<"__builtin_ia32_wbinvd">,
4822       Intrinsic<[], [], []>;
4823
4824   // Write back no-invalidate
4825   def int_x86_wbnoinvd : GCCBuiltin<"__builtin_ia32_wbnoinvd">,
4826       Intrinsic<[], [], []>;
4827 }
4828
4829 //===----------------------------------------------------------------------===//
4830 // Cache-line demote
4831
4832 let TargetPrefix = "x86" in {
4833   def int_x86_cldemote : GCCBuiltin<"__builtin_ia32_cldemote">,
4834       Intrinsic<[], [llvm_ptr_ty], []>;
4835 }
4836
4837 //===----------------------------------------------------------------------===//
4838 // Wait and pause enhancements
4839 let TargetPrefix = "x86" in {
4840   def int_x86_umonitor : GCCBuiltin<"__builtin_ia32_umonitor">,
4841               Intrinsic<[], [llvm_ptr_ty], []>;
4842   def int_x86_umwait : GCCBuiltin<"__builtin_ia32_umwait">,
4843               Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
4844   def int_x86_tpause : GCCBuiltin<"__builtin_ia32_tpause">,
4845               Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
4846 }
4847
4848 //===----------------------------------------------------------------------===//
4849 // Direct Move Instructions
4850
4851 let TargetPrefix = "x86" in {
4852   def int_x86_directstore32 : GCCBuiltin<"__builtin_ia32_directstore_u32">,
4853       Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], []>;
4854   def int_x86_directstore64 : GCCBuiltin<"__builtin_ia32_directstore_u64">,
4855       Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], []>;
4856   def int_x86_movdir64b : GCCBuiltin<"__builtin_ia32_movdir64b">,
4857       Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], []>;
4858 }
4859
4860 //===----------------------------------------------------------------------===//
4861 // PTWrite - Write data to processor trace pocket
4862
4863 let TargetPrefix = "x86" in {
4864   def int_x86_ptwrite32 : GCCBuiltin<"__builtin_ia32_ptwrite32">,
4865               Intrinsic<[], [llvm_i32_ty], []>;
4866   def int_x86_ptwrite64 : GCCBuiltin<"__builtin_ia32_ptwrite64">,
4867               Intrinsic<[], [llvm_i64_ty], []>;
4868 }
4869
4870 //===----------------------------------------------------------------------===//
4871 // INVPCID - Invalidate Process-Context Identifier
4872
4873 let TargetPrefix = "x86" in {
4874   def int_x86_invpcid : GCCBuiltin<"__builtin_ia32_invpcid">,
4875               Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>;
4876 }
4877
4878 let TargetPrefix = "x86" in {
4879   def int_x86_avx512bf16_cvtne2ps2bf16_128:
4880               GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_128">,
4881               Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_v4f32_ty],
4882               [IntrNoMem]>;
4883   def int_x86_avx512bf16_cvtne2ps2bf16_256:
4884               GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_256">,
4885               Intrinsic<[llvm_v16i16_ty], [llvm_v8f32_ty, llvm_v8f32_ty],
4886               [IntrNoMem]>;
4887   def int_x86_avx512bf16_cvtne2ps2bf16_512:
4888               GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_512">,
4889               Intrinsic<[llvm_v32i16_ty], [llvm_v16f32_ty, llvm_v16f32_ty],
4890               [IntrNoMem]>;
4891   // Intrinsic must be masked due to it producing less than 128 bits of results.
4892   def int_x86_avx512bf16_mask_cvtneps2bf16_128:
4893               Intrinsic<[llvm_v8i16_ty],
4894                         [llvm_v4f32_ty, llvm_v8i16_ty, llvm_v4i1_ty],
4895                         [IntrNoMem]>;
4896   def int_x86_avx512bf16_cvtneps2bf16_256:
4897               GCCBuiltin<"__builtin_ia32_cvtneps2bf16_256">,
4898               Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty], [IntrNoMem]>;
4899   def int_x86_avx512bf16_cvtneps2bf16_512:
4900               GCCBuiltin<"__builtin_ia32_cvtneps2bf16_512">,
4901               Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty], [IntrNoMem]>;
4902   def int_x86_avx512bf16_dpbf16ps_128:
4903               GCCBuiltin<"__builtin_ia32_dpbf16ps_128">,
4904               Intrinsic<[llvm_v4f32_ty],
4905               [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
4906   def int_x86_avx512bf16_dpbf16ps_256:
4907               GCCBuiltin<"__builtin_ia32_dpbf16ps_256">,
4908               Intrinsic<[llvm_v8f32_ty],
4909               [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>;
4910   def int_x86_avx512bf16_dpbf16ps_512:
4911               GCCBuiltin<"__builtin_ia32_dpbf16ps_512">,
4912               Intrinsic<[llvm_v16f32_ty],
4913               [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>;
4914 }
4915
4916 //===----------------------------------------------------------------------===//
4917 // ENQCMD - Enqueue Stores Instructions
4918
4919 let TargetPrefix = "x86" in {
4920   def int_x86_enqcmd : GCCBuiltin<"__builtin_ia32_enqcmd">,
4921               Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>;
4922   def int_x86_enqcmds : GCCBuiltin<"__builtin_ia32_enqcmds">,
4923               Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>;
4924 }