contrib/llvm/include/llvm/IR/IntrinsicsAArch64.td

   1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file defines all of the AARCH64-specific intrinsics.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 let TargetPrefix = "aarch64" in {
  15
  16 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  17 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  18 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  19 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  20
  21 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  22 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  23 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
  24                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  25 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
  26                                 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  27
  28 def int_aarch64_clrex : Intrinsic<[]>;
  29
  30 def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  31                                 LLVMMatchType<0>], [IntrNoMem]>;
  32 def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  33                                 LLVMMatchType<0>], [IntrNoMem]>;
  34
  35 //===----------------------------------------------------------------------===//
  36 // HINT
  37
  38 def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
  39
  40 //===----------------------------------------------------------------------===//
  41 // Data Barrier Instructions
  42
  43 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
  44 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
  45 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
  46
  47 // A space-consuming intrinsic primarily for testing block and jump table
  48 // placements. The first argument is the number of bytes this "instruction"
  49 // takes up, the second and return value are essentially chains, used to force
  50 // ordering during ISel.
  51 def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
  52
  53 }
  54
  55 //===----------------------------------------------------------------------===//
  56 // Advanced SIMD (NEON)
  57
  58 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  59   class AdvSIMD_2Scalar_Float_Intrinsic
  60     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  61                 [IntrNoMem]>;
  62
  63   class AdvSIMD_FPToIntRounding_Intrinsic
  64     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
  65
  66   class AdvSIMD_1IntArg_Intrinsic
  67     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  68   class AdvSIMD_1FloatArg_Intrinsic
  69     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  70   class AdvSIMD_1VectorArg_Intrinsic
  71     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  72   class AdvSIMD_1VectorArg_Expand_Intrinsic
  73     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  74   class AdvSIMD_1VectorArg_Long_Intrinsic
  75     : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
  76   class AdvSIMD_1IntArg_Narrow_Intrinsic
  77     : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
  78   class AdvSIMD_1VectorArg_Narrow_Intrinsic
  79     : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
  80   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
  81     : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  82   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
  83     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  84
  85   class AdvSIMD_2IntArg_Intrinsic
  86     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  87                 [IntrNoMem]>;
  88   class AdvSIMD_2FloatArg_Intrinsic
  89     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  90                 [IntrNoMem]>;
  91   class AdvSIMD_2VectorArg_Intrinsic
  92     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  93                 [IntrNoMem]>;
  94   class AdvSIMD_2VectorArg_Compare_Intrinsic
  95     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
  96                 [IntrNoMem]>;
  97   class AdvSIMD_2Arg_FloatCompare_Intrinsic
  98     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
  99                 [IntrNoMem]>;
 100   class AdvSIMD_2VectorArg_Long_Intrinsic
 101     : Intrinsic<[llvm_anyvector_ty],
 102                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
 103                 [IntrNoMem]>;
 104   class AdvSIMD_2VectorArg_Wide_Intrinsic
 105     : Intrinsic<[llvm_anyvector_ty],
 106                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
 107                 [IntrNoMem]>;
 108   class AdvSIMD_2VectorArg_Narrow_Intrinsic
 109     : Intrinsic<[llvm_anyvector_ty],
 110                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
 111                 [IntrNoMem]>;
 112   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
 113     : Intrinsic<[llvm_anyint_ty],
 114                 [LLVMExtendedType<0>, llvm_i32_ty],
 115                 [IntrNoMem]>;
 116   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
 117     : Intrinsic<[llvm_anyvector_ty],
 118                 [llvm_anyvector_ty],
 119                 [IntrNoMem]>;
 120   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
 121     : Intrinsic<[llvm_anyvector_ty],
 122                 [LLVMTruncatedType<0>],
 123                 [IntrNoMem]>;
 124   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
 125     : Intrinsic<[llvm_anyvector_ty],
 126                 [LLVMTruncatedType<0>, llvm_i32_ty],
 127                 [IntrNoMem]>;
 128   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
 129     : Intrinsic<[llvm_anyvector_ty],
 130                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
 131                 [IntrNoMem]>;
 132
 133   class AdvSIMD_3VectorArg_Intrinsic
 134       : Intrinsic<[llvm_anyvector_ty],
 135                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 136                [IntrNoMem]>;
 137   class AdvSIMD_3VectorArg_Scalar_Intrinsic
 138       : Intrinsic<[llvm_anyvector_ty],
 139                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
 140                [IntrNoMem]>;
 141   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
 142       : Intrinsic<[llvm_anyvector_ty],
 143                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
 144                 LLVMMatchType<1>], [IntrNoMem]>;
 145   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
 146     : Intrinsic<[llvm_anyvector_ty],
 147                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
 148                 [IntrNoMem]>;
 149   class AdvSIMD_CvtFxToFP_Intrinsic
 150     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
 151                 [IntrNoMem]>;
 152   class AdvSIMD_CvtFPToFx_Intrinsic
 153     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
 154                 [IntrNoMem]>;
 155
 156   class AdvSIMD_1Arg_Intrinsic
 157     : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 158
 159   class AdvSIMD_Dot_Intrinsic
 160     : Intrinsic<[llvm_anyvector_ty],
 161                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 162                 [IntrNoMem]>;
 163
 164   class AdvSIMD_FP16FML_Intrinsic
 165     : Intrinsic<[llvm_anyvector_ty],
 166                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 167                 [IntrNoMem]>;
 168 }
 169
 170 // Arithmetic ops
 171
 172 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
 173   // Vector Add Across Lanes
 174   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 175   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 176   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 177
 178   // Vector Long Add Across Lanes
 179   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 180   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 181
 182   // Vector Halving Add
 183   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
 184   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
 185
 186   // Vector Rounding Halving Add
 187   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
 188   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
 189
 190   // Vector Saturating Add
 191   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
 192   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
 193   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
 194   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
 195
 196   // Vector Add High-Half
 197   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 198   // header is no longer supported.
 199   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 200
 201   // Vector Rounding Add High-Half
 202   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 203
 204   // Vector Saturating Doubling Multiply High
 205   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
 206
 207   // Vector Saturating Rounding Doubling Multiply High
 208   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
 209
 210   // Vector Polynominal Multiply
 211   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
 212
 213   // Vector Long Multiply
 214   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
 215   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
 216   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 217
 218   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
 219   // it with a v16i8.
 220   def int_aarch64_neon_pmull64 :
 221         Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 222
 223   // Vector Extending Multiply
 224   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
 225     let IntrProperties = [IntrNoMem, Commutative];
 226   }
 227
 228   // Vector Saturating Doubling Long Multiply
 229   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 230   def int_aarch64_neon_sqdmulls_scalar
 231     : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 232
 233   // Vector Halving Subtract
 234   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
 235   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
 236
 237   // Vector Saturating Subtract
 238   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
 239   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
 240
 241   // Vector Subtract High-Half
 242   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 243   // header is no longer supported.
 244   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 245
 246   // Vector Rounding Subtract High-Half
 247   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 248
 249   // Vector Compare Absolute Greater-than-or-equal
 250   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 251
 252   // Vector Compare Absolute Greater-than
 253   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 254
 255   // Vector Absolute Difference
 256   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
 257   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
 258   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
 259
 260   // Scalar Absolute Difference
 261   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
 262
 263   // Vector Max
 264   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
 265   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
 266   def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
 267   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
 268
 269   // Vector Max Across Lanes
 270   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 271   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 272   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 273   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 274
 275   // Vector Min
 276   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
 277   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
 278   def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
 279   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
 280
 281   // Vector Min/Max Number
 282   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
 283   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
 284
 285   // Vector Min Across Lanes
 286   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 287   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 288   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 289   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 290
 291   // Pairwise Add
 292   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
 293
 294   // Long Pairwise Add
 295   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
 296   // uaddlp, but tblgen's type inference currently can't handle the
 297   // pattern fragments this ends up generating.
 298   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 299   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 300
 301   // Folding Maximum
 302   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
 303   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
 304   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
 305
 306   // Folding Minimum
 307   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
 308   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
 309   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
 310
 311   // Reciprocal Estimate/Step
 312   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
 313   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
 314
 315   // Reciprocal Exponent
 316   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
 317
 318   // Vector Saturating Shift Left
 319   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
 320   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
 321
 322   // Vector Rounding Shift Left
 323   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
 324   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
 325
 326   // Vector Saturating Rounding Shift Left
 327   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
 328   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
 329
 330   // Vector Signed->Unsigned Shift Left by Constant
 331   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
 332
 333   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
 334   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 335
 336   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
 337   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 338
 339   // Vector Narrowing Shift Right by Constant
 340   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 341   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 342
 343   // Vector Rounding Narrowing Shift Right by Constant
 344   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 345
 346   // Vector Rounding Narrowing Saturating Shift Right by Constant
 347   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 348   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 349
 350   // Vector Shift Left
 351   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
 352   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
 353
 354   // Vector Widening Shift Left by Constant
 355   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
 356   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 357   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 358
 359   // Vector Shift Right by Constant and Insert
 360   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 361
 362   // Vector Shift Left by Constant and Insert
 363   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 364
 365   // Vector Saturating Narrow
 366   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
 367   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
 368   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 369   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 370
 371   // Vector Saturating Extract and Unsigned Narrow
 372   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
 373   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 374
 375   // Vector Absolute Value
 376   def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
 377
 378   // Vector Saturating Absolute Value
 379   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
 380
 381   // Vector Saturating Negation
 382   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
 383
 384   // Vector Count Leading Sign Bits
 385   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
 386
 387   // Vector Reciprocal Estimate
 388   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
 389   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
 390
 391   // Vector Square Root Estimate
 392   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
 393   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
 394
 395   // Vector Bitwise Reverse
 396   def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
 397
 398   // Vector Conversions Between Half-Precision and Single-Precision.
 399   def int_aarch64_neon_vcvtfp2hf
 400     : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 401   def int_aarch64_neon_vcvthf2fp
 402     : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
 403
 404   // Vector Conversions Between Floating-point and Fixed-point.
 405   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
 406   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
 407   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 408   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 409
 410   // Vector FP->Int Conversions
 411   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
 412   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
 413   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
 414   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
 415   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
 416   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
 417   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
 418   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
 419   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
 420   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 421
 422   // Vector FP Rounding: only ties to even is unrepresented by a normal
 423   // intrinsic.
 424   def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
 425
 426   // Scalar FP->Int conversions
 427
 428   // Vector FP Inexact Narrowing
 429   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
 430
 431   // Scalar FP Inexact Narrowing
 432   def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
 433                                         [IntrNoMem]>;
 434
 435   // v8.2-A Dot Product
 436   def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
 437   def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
 438
 439   // v8.2-A FP16 Fused Multiply-Add Long
 440   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
 441   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
 442   def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
 443   def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
 444 }
 445
 446 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 447   class AdvSIMD_2Vector2Index_Intrinsic
 448     : Intrinsic<[llvm_anyvector_ty],
 449                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
 450                 [IntrNoMem]>;
 451 }
 452
 453 // Vector element to element moves
 454 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
 455
 456 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 457   class AdvSIMD_1Vec_Load_Intrinsic
 458       : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
 459                   [IntrReadMem, IntrArgMemOnly]>;
 460   class AdvSIMD_1Vec_Store_Lane_Intrinsic
 461     : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
 462                 [IntrArgMemOnly, NoCapture<2>]>;
 463
 464   class AdvSIMD_2Vec_Load_Intrinsic
 465     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
 466                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 467                 [IntrReadMem, IntrArgMemOnly]>;
 468   class AdvSIMD_2Vec_Load_Lane_Intrinsic
 469     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
 470                 [LLVMMatchType<0>, LLVMMatchType<0>,
 471                  llvm_i64_ty, llvm_anyptr_ty],
 472                 [IntrReadMem, IntrArgMemOnly]>;
 473   class AdvSIMD_2Vec_Store_Intrinsic
 474     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 475                      LLVMAnyPointerType<LLVMMatchType<0>>],
 476                 [IntrArgMemOnly, NoCapture<2>]>;
 477   class AdvSIMD_2Vec_Store_Lane_Intrinsic
 478     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 479                  llvm_i64_ty, llvm_anyptr_ty],
 480                 [IntrArgMemOnly, NoCapture<3>]>;
 481
 482   class AdvSIMD_3Vec_Load_Intrinsic
 483     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
 484                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 485                 [IntrReadMem, IntrArgMemOnly]>;
 486   class AdvSIMD_3Vec_Load_Lane_Intrinsic
 487     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
 488                 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
 489                  llvm_i64_ty, llvm_anyptr_ty],
 490                 [IntrReadMem, IntrArgMemOnly]>;
 491   class AdvSIMD_3Vec_Store_Intrinsic
 492     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 493                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
 494                 [IntrArgMemOnly, NoCapture<3>]>;
 495   class AdvSIMD_3Vec_Store_Lane_Intrinsic
 496     : Intrinsic<[], [llvm_anyvector_ty,
 497                  LLVMMatchType<0>, LLVMMatchType<0>,
 498                  llvm_i64_ty, llvm_anyptr_ty],
 499                 [IntrArgMemOnly, NoCapture<4>]>;
 500
 501   class AdvSIMD_4Vec_Load_Intrinsic
 502     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
 503                  LLVMMatchType<0>, LLVMMatchType<0>],
 504                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 505                 [IntrReadMem, IntrArgMemOnly]>;
 506   class AdvSIMD_4Vec_Load_Lane_Intrinsic
 507     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
 508                  LLVMMatchType<0>, LLVMMatchType<0>],
 509                 [LLVMMatchType<0>, LLVMMatchType<0>,
 510                  LLVMMatchType<0>, LLVMMatchType<0>,
 511                  llvm_i64_ty, llvm_anyptr_ty],
 512                 [IntrReadMem, IntrArgMemOnly]>;
 513   class AdvSIMD_4Vec_Store_Intrinsic
 514     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 515                  LLVMMatchType<0>, LLVMMatchType<0>,
 516                  LLVMAnyPointerType<LLVMMatchType<0>>],
 517                 [IntrArgMemOnly, NoCapture<4>]>;
 518   class AdvSIMD_4Vec_Store_Lane_Intrinsic
 519     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 520                  LLVMMatchType<0>, LLVMMatchType<0>,
 521                  llvm_i64_ty, llvm_anyptr_ty],
 522                 [IntrArgMemOnly, NoCapture<5>]>;
 523 }
 524
 525 // Memory ops
 526
 527 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
 528 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
 529 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
 530
 531 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
 532 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
 533 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
 534
 535 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
 536 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
 537 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
 538
 539 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
 540 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
 541 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
 542
 543 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
 544 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
 545 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
 546
 547 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
 548 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
 549 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
 550
 551 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
 552 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
 553 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
 554
 555 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 556   class AdvSIMD_Tbl1_Intrinsic
 557     : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
 558                 [IntrNoMem]>;
 559   class AdvSIMD_Tbl2_Intrinsic
 560     : Intrinsic<[llvm_anyvector_ty],
 561                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
 562   class AdvSIMD_Tbl3_Intrinsic
 563     : Intrinsic<[llvm_anyvector_ty],
 564                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 565                  LLVMMatchType<0>],
 566                 [IntrNoMem]>;
 567   class AdvSIMD_Tbl4_Intrinsic
 568     : Intrinsic<[llvm_anyvector_ty],
 569                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 570                  LLVMMatchType<0>],
 571                 [IntrNoMem]>;
 572
 573   class AdvSIMD_Tbx1_Intrinsic
 574     : Intrinsic<[llvm_anyvector_ty],
 575                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
 576                 [IntrNoMem]>;
 577   class AdvSIMD_Tbx2_Intrinsic
 578     : Intrinsic<[llvm_anyvector_ty],
 579                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 580                  LLVMMatchType<0>],
 581                 [IntrNoMem]>;
 582   class AdvSIMD_Tbx3_Intrinsic
 583     : Intrinsic<[llvm_anyvector_ty],
 584                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 585                  llvm_v16i8_ty, LLVMMatchType<0>],
 586                 [IntrNoMem]>;
 587   class AdvSIMD_Tbx4_Intrinsic
 588     : Intrinsic<[llvm_anyvector_ty],
 589                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 590                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
 591                 [IntrNoMem]>;
 592 }
 593 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
 594 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
 595 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
 596 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
 597
 598 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 599 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 600 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 601 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 602
 603 let TargetPrefix = "aarch64" in {
 604   class FPCR_Get_Intrinsic
 605     : Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>;
 606 }
 607
 608 // FPCR
 609 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
 610
 611 let TargetPrefix = "aarch64" in {
 612   class Crypto_AES_DataKey_Intrinsic
 613     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 614
 615   class Crypto_AES_Data_Intrinsic
 616     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 617
 618   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 619   // (v4i32).
 620   class Crypto_SHA_5Hash4Schedule_Intrinsic
 621     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
 622                 [IntrNoMem]>;
 623
 624   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 625   // (v4i32).
 626   class Crypto_SHA_1Hash_Intrinsic
 627     : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 628
 629   // SHA intrinsic taking 8 words of the schedule
 630   class Crypto_SHA_8Schedule_Intrinsic
 631     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 632
 633   // SHA intrinsic taking 12 words of the schedule
 634   class Crypto_SHA_12Schedule_Intrinsic
 635     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 636                 [IntrNoMem]>;
 637
 638   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
 639   class Crypto_SHA_8Hash4Schedule_Intrinsic
 640     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 641                 [IntrNoMem]>;
 642 }
 643
 644 // AES
 645 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
 646 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
 647 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
 648 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
 649
 650 // SHA1
 651 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 652 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 653 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 654 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
 655
 656 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
 657 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
 658
 659 // SHA256
 660 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
 661 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
 662 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
 663 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
 664
 665 //===----------------------------------------------------------------------===//
 666 // CRC32
 667
 668 let TargetPrefix = "aarch64" in {
 669
 670 def int_aarch64_crc32b  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 671     [IntrNoMem]>;
 672 def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 673     [IntrNoMem]>;
 674 def int_aarch64_crc32h  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 675     [IntrNoMem]>;
 676 def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 677     [IntrNoMem]>;
 678 def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 679     [IntrNoMem]>;
 680 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 681     [IntrNoMem]>;
 682 def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 683     [IntrNoMem]>;
 684 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 685     [IntrNoMem]>;
 686 }