contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsAArch64.td

   1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines all of the AARCH64-specific intrinsics.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 let TargetPrefix = "aarch64" in {
  14
  15 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  16 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  17 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  18 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  19
  20 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  21 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  22 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
  23                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  24 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
  25                                 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  26
  27 def int_aarch64_clrex : Intrinsic<[]>;
  28
  29 def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  30                                 LLVMMatchType<0>], [IntrNoMem]>;
  31 def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  32                                 LLVMMatchType<0>], [IntrNoMem]>;
  33
  34 def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
  35
  36 def int_aarch64_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
  37 def int_aarch64_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
  38
  39 //===----------------------------------------------------------------------===//
  40 // HINT
  41
  42 def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
  43
  44 //===----------------------------------------------------------------------===//
  45 // Data Barrier Instructions
  46
  47 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
  48 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
  49 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
  50
  51 // A space-consuming intrinsic primarily for testing block and jump table
  52 // placements. The first argument is the number of bytes this "instruction"
  53 // takes up, the second and return value are essentially chains, used to force
  54 // ordering during ISel.
  55 def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
  56
  57 }
  58
  59 //===----------------------------------------------------------------------===//
  60 // Advanced SIMD (NEON)
  61
  62 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  63   class AdvSIMD_2Scalar_Float_Intrinsic
  64     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  65                 [IntrNoMem]>;
  66
  67   class AdvSIMD_FPToIntRounding_Intrinsic
  68     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
  69
  70   class AdvSIMD_1IntArg_Intrinsic
  71     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  72   class AdvSIMD_1FloatArg_Intrinsic
  73     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  74   class AdvSIMD_1VectorArg_Intrinsic
  75     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  76   class AdvSIMD_1VectorArg_Expand_Intrinsic
  77     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  78   class AdvSIMD_1VectorArg_Long_Intrinsic
  79     : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
  80   class AdvSIMD_1IntArg_Narrow_Intrinsic
  81     : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
  82   class AdvSIMD_1VectorArg_Narrow_Intrinsic
  83     : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
  84   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
  85     : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  86   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
  87     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  88
  89   class AdvSIMD_2IntArg_Intrinsic
  90     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  91                 [IntrNoMem]>;
  92   class AdvSIMD_2FloatArg_Intrinsic
  93     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  94                 [IntrNoMem]>;
  95   class AdvSIMD_2VectorArg_Intrinsic
  96     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  97                 [IntrNoMem]>;
  98   class AdvSIMD_2VectorArg_Compare_Intrinsic
  99     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
 100                 [IntrNoMem]>;
 101   class AdvSIMD_2Arg_FloatCompare_Intrinsic
 102     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
 103                 [IntrNoMem]>;
 104   class AdvSIMD_2VectorArg_Long_Intrinsic
 105     : Intrinsic<[llvm_anyvector_ty],
 106                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
 107                 [IntrNoMem]>;
 108   class AdvSIMD_2VectorArg_Wide_Intrinsic
 109     : Intrinsic<[llvm_anyvector_ty],
 110                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
 111                 [IntrNoMem]>;
 112   class AdvSIMD_2VectorArg_Narrow_Intrinsic
 113     : Intrinsic<[llvm_anyvector_ty],
 114                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
 115                 [IntrNoMem]>;
 116   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
 117     : Intrinsic<[llvm_anyint_ty],
 118                 [LLVMExtendedType<0>, llvm_i32_ty],
 119                 [IntrNoMem]>;
 120   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
 121     : Intrinsic<[llvm_anyvector_ty],
 122                 [llvm_anyvector_ty],
 123                 [IntrNoMem]>;
 124   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
 125     : Intrinsic<[llvm_anyvector_ty],
 126                 [LLVMTruncatedType<0>],
 127                 [IntrNoMem]>;
 128   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
 129     : Intrinsic<[llvm_anyvector_ty],
 130                 [LLVMTruncatedType<0>, llvm_i32_ty],
 131                 [IntrNoMem]>;
 132   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
 133     : Intrinsic<[llvm_anyvector_ty],
 134                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
 135                 [IntrNoMem]>;
 136   class AdvSIMD_2VectorArg_Lane_Intrinsic
 137     : Intrinsic<[llvm_anyint_ty],
 138                 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
 139                 [IntrNoMem]>;
 140
 141   class AdvSIMD_3VectorArg_Intrinsic
 142       : Intrinsic<[llvm_anyvector_ty],
 143                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 144                [IntrNoMem]>;
 145   class AdvSIMD_3VectorArg_Scalar_Intrinsic
 146       : Intrinsic<[llvm_anyvector_ty],
 147                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
 148                [IntrNoMem]>;
 149   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
 150       : Intrinsic<[llvm_anyvector_ty],
 151                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
 152                 LLVMMatchType<1>], [IntrNoMem]>;
 153   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
 154     : Intrinsic<[llvm_anyvector_ty],
 155                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
 156                 [IntrNoMem]>;
 157   class AdvSIMD_CvtFxToFP_Intrinsic
 158     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
 159                 [IntrNoMem]>;
 160   class AdvSIMD_CvtFPToFx_Intrinsic
 161     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
 162                 [IntrNoMem]>;
 163
 164   class AdvSIMD_1Arg_Intrinsic
 165     : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 166
 167   class AdvSIMD_Dot_Intrinsic
 168     : Intrinsic<[llvm_anyvector_ty],
 169                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 170                 [IntrNoMem]>;
 171
 172   class AdvSIMD_FP16FML_Intrinsic
 173     : Intrinsic<[llvm_anyvector_ty],
 174                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 175                 [IntrNoMem]>;
 176
 177   class AdvSIMD_MatMul_Intrinsic
 178     : Intrinsic<[llvm_anyvector_ty],
 179                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 180                 [IntrNoMem]>;
 181
 182   class AdvSIMD_FML_Intrinsic
 183     : Intrinsic<[llvm_anyvector_ty],
 184                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 185                 [IntrNoMem]>;
 186
 187 }
 188
 189 // Arithmetic ops
 190
 191 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
 192   // Vector Add Across Lanes
 193   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 194   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 195   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 196
 197   // Vector Long Add Across Lanes
 198   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 199   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 200
 201   // Vector Halving Add
 202   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
 203   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
 204
 205   // Vector Rounding Halving Add
 206   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
 207   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
 208
 209   // Vector Saturating Add
 210   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
 211   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
 212   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
 213   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
 214
 215   // Vector Add High-Half
 216   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 217   // header is no longer supported.
 218   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 219
 220   // Vector Rounding Add High-Half
 221   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 222
 223   // Vector Saturating Doubling Multiply High
 224   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
 225   def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
 226   def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
 227
 228   // Vector Saturating Rounding Doubling Multiply High
 229   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
 230   def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
 231   def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
 232
 233   // Vector Polynominal Multiply
 234   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
 235
 236   // Vector Long Multiply
 237   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
 238   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
 239   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 240
 241   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
 242   // it with a v16i8.
 243   def int_aarch64_neon_pmull64 :
 244         Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 245
 246   // Vector Extending Multiply
 247   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
 248     let IntrProperties = [IntrNoMem, Commutative];
 249   }
 250
 251   // Vector Saturating Doubling Long Multiply
 252   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 253   def int_aarch64_neon_sqdmulls_scalar
 254     : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 255
 256   // Vector Halving Subtract
 257   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
 258   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
 259
 260   // Vector Saturating Subtract
 261   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
 262   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
 263
 264   // Vector Subtract High-Half
 265   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 266   // header is no longer supported.
 267   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 268
 269   // Vector Rounding Subtract High-Half
 270   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 271
 272   // Vector Compare Absolute Greater-than-or-equal
 273   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 274
 275   // Vector Compare Absolute Greater-than
 276   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 277
 278   // Vector Absolute Difference
 279   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
 280   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
 281   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
 282
 283   // Scalar Absolute Difference
 284   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
 285
 286   // Vector Max
 287   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
 288   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
 289   def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
 290   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
 291
 292   // Vector Max Across Lanes
 293   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 294   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 295   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 296   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 297
 298   // Vector Min
 299   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
 300   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
 301   def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
 302   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
 303
 304   // Vector Min/Max Number
 305   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
 306   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
 307
 308   // Vector Min Across Lanes
 309   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 310   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 311   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 312   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 313
 314   // Pairwise Add
 315   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
 316   def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
 317
 318   // Long Pairwise Add
 319   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
 320   // uaddlp, but tblgen's type inference currently can't handle the
 321   // pattern fragments this ends up generating.
 322   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 323   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 324
 325   // Folding Maximum
 326   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
 327   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
 328   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
 329
 330   // Folding Minimum
 331   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
 332   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
 333   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
 334
 335   // Reciprocal Estimate/Step
 336   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
 337   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
 338
 339   // Reciprocal Exponent
 340   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
 341
 342   // Vector Saturating Shift Left
 343   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
 344   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
 345
 346   // Vector Rounding Shift Left
 347   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
 348   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
 349
 350   // Vector Saturating Rounding Shift Left
 351   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
 352   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
 353
 354   // Vector Signed->Unsigned Shift Left by Constant
 355   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
 356
 357   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
 358   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 359
 360   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
 361   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 362
 363   // Vector Narrowing Shift Right by Constant
 364   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 365   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 366
 367   // Vector Rounding Narrowing Shift Right by Constant
 368   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 369
 370   // Vector Rounding Narrowing Saturating Shift Right by Constant
 371   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 372   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 373
 374   // Vector Shift Left
 375   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
 376   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
 377
 378   // Vector Widening Shift Left by Constant
 379   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
 380   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 381   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 382
 383   // Vector Shift Right by Constant and Insert
 384   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 385
 386   // Vector Shift Left by Constant and Insert
 387   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 388
 389   // Vector Saturating Narrow
 390   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
 391   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
 392   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 393   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 394
 395   // Vector Saturating Extract and Unsigned Narrow
 396   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
 397   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 398
 399   // Vector Absolute Value
 400   def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
 401
 402   // Vector Saturating Absolute Value
 403   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
 404
 405   // Vector Saturating Negation
 406   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
 407
 408   // Vector Count Leading Sign Bits
 409   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
 410
 411   // Vector Reciprocal Estimate
 412   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
 413   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
 414
 415   // Vector Square Root Estimate
 416   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
 417   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
 418
 419   // Vector Bitwise Reverse
 420   def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
 421
 422   // Vector Conversions Between Half-Precision and Single-Precision.
 423   def int_aarch64_neon_vcvtfp2hf
 424     : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 425   def int_aarch64_neon_vcvthf2fp
 426     : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
 427
 428   // Vector Conversions Between Floating-point and Fixed-point.
 429   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
 430   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
 431   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 432   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 433
 434   // Vector FP->Int Conversions
 435   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
 436   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
 437   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
 438   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
 439   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
 440   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
 441   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
 442   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
 443   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
 444   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 445
 446   // Vector FP Rounding: only ties to even is unrepresented by a normal
 447   // intrinsic.
 448   def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
 449
 450   // Scalar FP->Int conversions
 451
 452   // Vector FP Inexact Narrowing
 453   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
 454
 455   // Scalar FP Inexact Narrowing
 456   def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
 457                                         [IntrNoMem]>;
 458
 459   // v8.2-A Dot Product
 460   def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
 461   def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
 462
 463 // v8.6-A Matrix Multiply Intrinsics
 464   def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
 465   def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
 466   def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
 467   def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
 468   def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
 469   def int_aarch64_neon_bfmmla : AdvSIMD_MatMul_Intrinsic;
 470   def int_aarch64_neon_bfmlalb : AdvSIMD_FML_Intrinsic;
 471   def int_aarch64_neon_bfmlalt : AdvSIMD_FML_Intrinsic;
 472
 473
 474   // v8.6-A Bfloat Intrinsics
 475   def int_aarch64_neon_bfcvt
 476     : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
 477   def int_aarch64_neon_bfcvtn
 478     : Intrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 479   def int_aarch64_neon_bfcvtn2
 480     : Intrinsic<[llvm_v8bf16_ty],
 481                 [llvm_v8bf16_ty, llvm_v4f32_ty],
 482                 [IntrNoMem]>;
 483
 484   // v8.2-A FP16 Fused Multiply-Add Long
 485   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
 486   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
 487   def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
 488   def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
 489
 490   // v8.3-A Floating-point complex add
 491   def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
 492   def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
 493 }
 494
 495 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 496   class AdvSIMD_2Vector2Index_Intrinsic
 497     : Intrinsic<[llvm_anyvector_ty],
 498                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
 499                 [IntrNoMem]>;
 500 }
 501
 502 // Vector element to element moves
 503 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
 504
 505 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 506   class AdvSIMD_1Vec_Load_Intrinsic
 507       : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
 508                   [IntrReadMem, IntrArgMemOnly]>;
 509   class AdvSIMD_1Vec_Store_Lane_Intrinsic
 510     : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
 511                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 512
 513   class AdvSIMD_2Vec_Load_Intrinsic
 514     : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
 515                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 516                 [IntrReadMem, IntrArgMemOnly]>;
 517   class AdvSIMD_2Vec_Load_Lane_Intrinsic
 518     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
 519                 [LLVMMatchType<0>, llvm_anyvector_ty,
 520                  llvm_i64_ty, llvm_anyptr_ty],
 521                 [IntrReadMem, IntrArgMemOnly]>;
 522   class AdvSIMD_2Vec_Store_Intrinsic
 523     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 524                      LLVMAnyPointerType<LLVMMatchType<0>>],
 525                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 526   class AdvSIMD_2Vec_Store_Lane_Intrinsic
 527     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 528                  llvm_i64_ty, llvm_anyptr_ty],
 529                 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 530
 531   class AdvSIMD_3Vec_Load_Intrinsic
 532     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
 533                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 534                 [IntrReadMem, IntrArgMemOnly]>;
 535   class AdvSIMD_3Vec_Load_Lane_Intrinsic
 536     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 537                 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
 538                  llvm_i64_ty, llvm_anyptr_ty],
 539                 [IntrReadMem, IntrArgMemOnly]>;
 540   class AdvSIMD_3Vec_Store_Intrinsic
 541     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 542                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
 543                 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 544   class AdvSIMD_3Vec_Store_Lane_Intrinsic
 545     : Intrinsic<[], [llvm_anyvector_ty,
 546                  LLVMMatchType<0>, LLVMMatchType<0>,
 547                  llvm_i64_ty, llvm_anyptr_ty],
 548                 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 549
 550   class AdvSIMD_4Vec_Load_Intrinsic
 551     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 552                  LLVMMatchType<0>, llvm_anyvector_ty],
 553                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 554                 [IntrReadMem, IntrArgMemOnly]>;
 555   class AdvSIMD_4Vec_Load_Lane_Intrinsic
 556     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 557                  LLVMMatchType<0>, LLVMMatchType<0>],
 558                 [LLVMMatchType<0>, LLVMMatchType<0>,
 559                  LLVMMatchType<0>, llvm_anyvector_ty,
 560                  llvm_i64_ty, llvm_anyptr_ty],
 561                 [IntrReadMem, IntrArgMemOnly]>;
 562   class AdvSIMD_4Vec_Store_Intrinsic
 563     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 564                  LLVMMatchType<0>, LLVMMatchType<0>,
 565                  LLVMAnyPointerType<LLVMMatchType<0>>],
 566                 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 567   class AdvSIMD_4Vec_Store_Lane_Intrinsic
 568     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 569                  LLVMMatchType<0>, LLVMMatchType<0>,
 570                  llvm_i64_ty, llvm_anyptr_ty],
 571                 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 572 }
 573
 574 // Memory ops
 575
 576 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
 577 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
 578 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
 579
 580 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
 581 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
 582 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
 583
 584 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
 585 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
 586 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
 587
 588 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
 589 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
 590 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
 591
 592 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
 593 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
 594 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
 595
 596 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
 597 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
 598 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
 599
 600 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
 601 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
 602 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
 603
 604 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 605   class AdvSIMD_Tbl1_Intrinsic
 606     : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
 607                 [IntrNoMem]>;
 608   class AdvSIMD_Tbl2_Intrinsic
 609     : Intrinsic<[llvm_anyvector_ty],
 610                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
 611   class AdvSIMD_Tbl3_Intrinsic
 612     : Intrinsic<[llvm_anyvector_ty],
 613                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 614                  LLVMMatchType<0>],
 615                 [IntrNoMem]>;
 616   class AdvSIMD_Tbl4_Intrinsic
 617     : Intrinsic<[llvm_anyvector_ty],
 618                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 619                  LLVMMatchType<0>],
 620                 [IntrNoMem]>;
 621
 622   class AdvSIMD_Tbx1_Intrinsic
 623     : Intrinsic<[llvm_anyvector_ty],
 624                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
 625                 [IntrNoMem]>;
 626   class AdvSIMD_Tbx2_Intrinsic
 627     : Intrinsic<[llvm_anyvector_ty],
 628                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 629                  LLVMMatchType<0>],
 630                 [IntrNoMem]>;
 631   class AdvSIMD_Tbx3_Intrinsic
 632     : Intrinsic<[llvm_anyvector_ty],
 633                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 634                  llvm_v16i8_ty, LLVMMatchType<0>],
 635                 [IntrNoMem]>;
 636   class AdvSIMD_Tbx4_Intrinsic
 637     : Intrinsic<[llvm_anyvector_ty],
 638                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 639                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
 640                 [IntrNoMem]>;
 641 }
 642 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
 643 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
 644 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
 645 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
 646
 647 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 648 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 649 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 650 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 651
 652 let TargetPrefix = "aarch64" in {
 653   class FPCR_Get_Intrinsic
 654     : Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
 655 }
 656
 657 // FPCR
 658 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
 659
 660 let TargetPrefix = "aarch64" in {
 661   class Crypto_AES_DataKey_Intrinsic
 662     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 663
 664   class Crypto_AES_Data_Intrinsic
 665     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 666
 667   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 668   // (v4i32).
 669   class Crypto_SHA_5Hash4Schedule_Intrinsic
 670     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
 671                 [IntrNoMem]>;
 672
 673   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 674   // (v4i32).
 675   class Crypto_SHA_1Hash_Intrinsic
 676     : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 677
 678   // SHA intrinsic taking 8 words of the schedule
 679   class Crypto_SHA_8Schedule_Intrinsic
 680     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 681
 682   // SHA intrinsic taking 12 words of the schedule
 683   class Crypto_SHA_12Schedule_Intrinsic
 684     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 685                 [IntrNoMem]>;
 686
 687   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
 688   class Crypto_SHA_8Hash4Schedule_Intrinsic
 689     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 690                 [IntrNoMem]>;
 691 }
 692
 693 // AES
 694 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
 695 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
 696 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
 697 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
 698
 699 // SHA1
 700 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 701 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 702 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 703 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
 704
 705 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
 706 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
 707
 708 // SHA256
 709 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
 710 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
 711 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
 712 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
 713
 714 //===----------------------------------------------------------------------===//
 715 // CRC32
 716
 717 let TargetPrefix = "aarch64" in {
 718
 719 def int_aarch64_crc32b  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 720     [IntrNoMem]>;
 721 def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 722     [IntrNoMem]>;
 723 def int_aarch64_crc32h  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 724     [IntrNoMem]>;
 725 def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 726     [IntrNoMem]>;
 727 def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 728     [IntrNoMem]>;
 729 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 730     [IntrNoMem]>;
 731 def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 732     [IntrNoMem]>;
 733 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 734     [IntrNoMem]>;
 735 }
 736
 737 //===----------------------------------------------------------------------===//
 738 // Memory Tagging Extensions (MTE) Intrinsics
 739 let TargetPrefix = "aarch64" in {
 740 def int_aarch64_irg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 741     [IntrNoMem, IntrHasSideEffects]>;
 742 def int_aarch64_addg  : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 743     [IntrNoMem]>;
 744 def int_aarch64_gmi   : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
 745     [IntrNoMem]>;
 746 def int_aarch64_ldg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
 747     [IntrReadMem]>;
 748 def int_aarch64_stg   : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
 749     [IntrWriteMem]>;
 750 def int_aarch64_subp :  Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
 751     [IntrNoMem]>;
 752
 753 // The following are codegen-only intrinsics for stack instrumentation.
 754
 755 // Generate a randomly tagged stack base pointer.
 756 def int_aarch64_irg_sp   : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty],
 757     [IntrNoMem, IntrHasSideEffects]>;
 758
 759 // Transfer pointer tag with offset.
 760 // ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
 761 // * address is the address in ptr0
 762 // * tag is a function of (tag in baseptr, tag_offset).
 763 // Address bits in baseptr and tag bits in ptr0 are ignored.
 764 // When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
 765 //   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
 766 // It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
 767 def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
 768     [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 769
 770 // Update allocation tags for the memory range to match the tag in the pointer argument.
 771 def int_aarch64_settag  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 772     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 773
 774 // Update allocation tags for the memory range to match the tag in the pointer argument,
 775 // and set memory contents to zero.
 776 def int_aarch64_settag_zero  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 777     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 778
 779 // Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
 780 def int_aarch64_stgp  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
 781     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 782 }
 783
 784 // Transactional Memory Extension (TME) Intrinsics
 785 let TargetPrefix = "aarch64" in {
 786 def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
 787                          Intrinsic<[llvm_i64_ty]>;
 788
 789 def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>;
 790
 791 def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
 792                           Intrinsic<[], [llvm_i64_ty], [ImmArg<ArgIndex<0>>]>;
 793
 794 def int_aarch64_ttest   : GCCBuiltin<"__builtin_arm_ttest">,
 795                           Intrinsic<[llvm_i64_ty], [],
 796                                     [IntrNoMem, IntrHasSideEffects]>;
 797 }
 798
 799 def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
 800 def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
 801 def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
 802 def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
 803 def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
 804 def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
 805 def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
 806 def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
 807 def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
 808 def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
 809 def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
 810
 811 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 812
 813   class AdvSIMD_SVE_Create_2Vector_Tuple
 814     : Intrinsic<[llvm_anyvector_ty],
 815                 [llvm_anyvector_ty, LLVMMatchType<1>],
 816                 [IntrReadMem]>;
 817
 818   class AdvSIMD_SVE_Create_3Vector_Tuple
 819     : Intrinsic<[llvm_anyvector_ty],
 820                 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
 821                 [IntrReadMem]>;
 822
 823   class AdvSIMD_SVE_Create_4Vector_Tuple
 824     : Intrinsic<[llvm_anyvector_ty],
 825                 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
 826                  LLVMMatchType<1>],
 827                 [IntrReadMem]>;
 828
 829   class AdvSIMD_SVE_Set_Vector_Tuple
 830     : Intrinsic<[llvm_anyvector_ty],
 831                 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
 832                 [IntrReadMem, ImmArg<ArgIndex<1>>]>;
 833
 834   class AdvSIMD_SVE_Get_Vector_Tuple
 835     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
 836                 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 837
 838   class AdvSIMD_ManyVec_PredLoad_Intrinsic
 839     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
 840                 [IntrReadMem, IntrArgMemOnly]>;
 841
 842   class AdvSIMD_1Vec_PredLoad_Intrinsic
 843     : Intrinsic<[llvm_anyvector_ty],
 844                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 845                  LLVMPointerToElt<0>],
 846                 [IntrReadMem, IntrArgMemOnly]>;
 847
 848   class AdvSIMD_1Vec_PredStore_Intrinsic
 849     : Intrinsic<[],
 850                 [llvm_anyvector_ty,
 851                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 852                  LLVMPointerToElt<0>],
 853                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 854
 855   class AdvSIMD_2Vec_PredStore_Intrinsic
 856       : Intrinsic<[],
 857                   [llvm_anyvector_ty, LLVMMatchType<0>,
 858                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
 859                   [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 860
 861   class AdvSIMD_3Vec_PredStore_Intrinsic
 862       : Intrinsic<[],
 863                   [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
 864                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
 865                   [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 866
 867   class AdvSIMD_4Vec_PredStore_Intrinsic
 868       : Intrinsic<[],
 869                   [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
 870                    LLVMMatchType<0>,
 871                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
 872                   [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 873
 874   class AdvSIMD_SVE_Index_Intrinsic
 875     : Intrinsic<[llvm_anyvector_ty],
 876                 [LLVMVectorElementType<0>,
 877                  LLVMVectorElementType<0>],
 878                 [IntrNoMem]>;
 879
 880   class AdvSIMD_Merged1VectorArg_Intrinsic
 881     : Intrinsic<[llvm_anyvector_ty],
 882                 [LLVMMatchType<0>,
 883                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 884                  LLVMMatchType<0>],
 885                 [IntrNoMem]>;
 886
 887   class AdvSIMD_2VectorArgIndexed_Intrinsic
 888     : Intrinsic<[llvm_anyvector_ty],
 889                 [LLVMMatchType<0>,
 890                  LLVMMatchType<0>,
 891                  llvm_i32_ty],
 892                 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 893
 894   class AdvSIMD_3VectorArgIndexed_Intrinsic
 895     : Intrinsic<[llvm_anyvector_ty],
 896                 [LLVMMatchType<0>,
 897                  LLVMMatchType<0>,
 898                  LLVMMatchType<0>,
 899                  llvm_i32_ty],
 900                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
 901
 902   class AdvSIMD_Pred1VectorArg_Intrinsic
 903     : Intrinsic<[llvm_anyvector_ty],
 904                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 905                  LLVMMatchType<0>],
 906                 [IntrNoMem]>;
 907
 908   class AdvSIMD_Pred2VectorArg_Intrinsic
 909     : Intrinsic<[llvm_anyvector_ty],
 910                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 911                  LLVMMatchType<0>,
 912                  LLVMMatchType<0>],
 913                 [IntrNoMem]>;
 914
 915   class AdvSIMD_Pred3VectorArg_Intrinsic
 916     : Intrinsic<[llvm_anyvector_ty],
 917                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 918                  LLVMMatchType<0>,
 919                  LLVMMatchType<0>,
 920                  LLVMMatchType<0>],
 921                 [IntrNoMem]>;
 922
 923   class AdvSIMD_SVE_Compare_Intrinsic
 924     : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
 925                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 926                  llvm_anyvector_ty,
 927                  LLVMMatchType<0>],
 928                 [IntrNoMem]>;
 929
 930   class AdvSIMD_SVE_CompareWide_Intrinsic
 931     : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
 932                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 933                  llvm_anyvector_ty,
 934                  llvm_nxv2i64_ty],
 935                 [IntrNoMem]>;
 936
 937   class AdvSIMD_SVE_Saturating_Intrinsic
 938     : Intrinsic<[llvm_anyvector_ty],
 939                 [LLVMMatchType<0>,
 940                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
 941                 [IntrNoMem]>;
 942
 943   class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
 944     : Intrinsic<[llvm_anyvector_ty],
 945                 [LLVMMatchType<0>,
 946                  llvm_i32_ty,
 947                  llvm_i32_ty],
 948                 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
 949
 950   class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
 951     : Intrinsic<[T],
 952                 [T, llvm_anyvector_ty],
 953                 [IntrNoMem]>;
 954
 955   class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
 956     : Intrinsic<[T],
 957                 [T, llvm_i32_ty, llvm_i32_ty],
 958                 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
 959
 960   class AdvSIMD_SVE_CNT_Intrinsic
 961     : Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
 962                 [LLVMVectorOfBitcastsToInt<0>,
 963                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 964                  llvm_anyvector_ty],
 965                 [IntrNoMem]>;
 966
 967   class AdvSIMD_SVE_ReduceWithInit_Intrinsic
 968     : Intrinsic<[LLVMVectorElementType<0>],
 969                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 970                  LLVMVectorElementType<0>,
 971                  llvm_anyvector_ty],
 972                 [IntrNoMem]>;
 973
 974   class AdvSIMD_SVE_ShiftByImm_Intrinsic
 975     : Intrinsic<[llvm_anyvector_ty],
 976                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 977                  LLVMMatchType<0>,
 978                  llvm_i32_ty],
 979                 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 980
 981   class AdvSIMD_SVE_ShiftWide_Intrinsic
 982     : Intrinsic<[llvm_anyvector_ty],
 983                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 984                  LLVMMatchType<0>,
 985                  llvm_nxv2i64_ty],
 986                 [IntrNoMem]>;
 987
 988   class AdvSIMD_SVE_Unpack_Intrinsic
 989     : Intrinsic<[llvm_anyvector_ty],
 990                [LLVMSubdivide2VectorType<0>],
 991                [IntrNoMem]>;
 992
 993   class AdvSIMD_SVE_CADD_Intrinsic
 994     : Intrinsic<[llvm_anyvector_ty],
 995                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 996                  LLVMMatchType<0>,
 997                  LLVMMatchType<0>,
 998                  llvm_i32_ty],
 999                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1000
1001   class AdvSIMD_SVE_CMLA_Intrinsic
1002     : Intrinsic<[llvm_anyvector_ty],
1003                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1004                  LLVMMatchType<0>,
1005                  LLVMMatchType<0>,
1006                  LLVMMatchType<0>,
1007                  llvm_i32_ty],
1008                 [IntrNoMem, ImmArg<ArgIndex<4>>]>;
1009
1010   class AdvSIMD_SVE_CMLA_LANE_Intrinsic
1011     : Intrinsic<[llvm_anyvector_ty],
1012                 [LLVMMatchType<0>,
1013                  LLVMMatchType<0>,
1014                  LLVMMatchType<0>,
1015                  llvm_i32_ty,
1016                  llvm_i32_ty],
1017                 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1018
1019   class AdvSIMD_SVE_DUP_Intrinsic
1020     : Intrinsic<[llvm_anyvector_ty],
1021                 [LLVMMatchType<0>,
1022                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1023                  LLVMVectorElementType<0>],
1024                 [IntrNoMem]>;
1025
1026   class AdvSIMD_SVE_DUP_Unpred_Intrinsic
1027     : Intrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
1028                 [IntrNoMem]>;
1029
1030   class AdvSIMD_SVE_DUPQ_Intrinsic
1031     : Intrinsic<[llvm_anyvector_ty],
1032                 [LLVMMatchType<0>,
1033                  llvm_i64_ty],
1034                 [IntrNoMem]>;
1035
1036   class AdvSIMD_SVE_EXPA_Intrinsic
1037     : Intrinsic<[llvm_anyvector_ty],
1038                 [LLVMVectorOfBitcastsToInt<0>],
1039                 [IntrNoMem]>;
1040
1041   class AdvSIMD_SVE_FCVT_Intrinsic
1042     : Intrinsic<[llvm_anyvector_ty],
1043                 [LLVMMatchType<0>,
1044                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1045                  llvm_anyvector_ty],
1046                 [IntrNoMem]>;
1047
1048   class AdvSIMD_SVE_FCVTZS_Intrinsic
1049     : Intrinsic<[llvm_anyvector_ty],
1050                 [LLVMVectorOfBitcastsToInt<0>,
1051                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1052                  llvm_anyvector_ty],
1053                 [IntrNoMem]>;
1054
1055   class AdvSIMD_SVE_INSR_Intrinsic
1056     : Intrinsic<[llvm_anyvector_ty],
1057                 [LLVMMatchType<0>,
1058                  LLVMVectorElementType<0>],
1059                 [IntrNoMem]>;
1060
1061   class AdvSIMD_SVE_PTRUE_Intrinsic
1062     : Intrinsic<[llvm_anyvector_ty],
1063                 [llvm_i32_ty],
1064                 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1065
1066   class AdvSIMD_SVE_PUNPKHI_Intrinsic
1067     : Intrinsic<[LLVMHalfElementsVectorType<0>],
1068                 [llvm_anyvector_ty],
1069                 [IntrNoMem]>;
1070
1071   class AdvSIMD_SVE_SCALE_Intrinsic
1072     : Intrinsic<[llvm_anyvector_ty],
1073                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1074                  LLVMMatchType<0>,
1075                  LLVMVectorOfBitcastsToInt<0>],
1076                 [IntrNoMem]>;
1077
1078   class AdvSIMD_SVE_SCVTF_Intrinsic
1079     : Intrinsic<[llvm_anyvector_ty],
1080                 [LLVMMatchType<0>,
1081                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1082                  llvm_anyvector_ty],
1083                 [IntrNoMem]>;
1084
1085   class AdvSIMD_SVE_TSMUL_Intrinsic
1086     : Intrinsic<[llvm_anyvector_ty],
1087                 [LLVMMatchType<0>,
1088                  LLVMVectorOfBitcastsToInt<0>],
1089                 [IntrNoMem]>;
1090
1091   class AdvSIMD_SVE_CNTB_Intrinsic
1092     : Intrinsic<[llvm_i64_ty],
1093                 [llvm_i32_ty],
1094                 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1095
1096   class AdvSIMD_SVE_CNTP_Intrinsic
1097     : Intrinsic<[llvm_i64_ty],
1098                 [llvm_anyvector_ty, LLVMMatchType<0>],
1099                 [IntrNoMem]>;
1100
1101   class AdvSIMD_SVE_DOT_Intrinsic
1102     : Intrinsic<[llvm_anyvector_ty],
1103                 [LLVMMatchType<0>,
1104                  LLVMSubdivide4VectorType<0>,
1105                  LLVMSubdivide4VectorType<0>],
1106                 [IntrNoMem]>;
1107
1108   class AdvSIMD_SVE_DOT_Indexed_Intrinsic
1109     : Intrinsic<[llvm_anyvector_ty],
1110                 [LLVMMatchType<0>,
1111                  LLVMSubdivide4VectorType<0>,
1112                  LLVMSubdivide4VectorType<0>,
1113                  llvm_i32_ty],
1114                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1115
1116   class AdvSIMD_SVE_PTEST_Intrinsic
1117     : Intrinsic<[llvm_i1_ty],
1118                 [llvm_anyvector_ty,
1119                  LLVMMatchType<0>],
1120                 [IntrNoMem]>;
1121
1122   class AdvSIMD_SVE_TBL_Intrinsic
1123     : Intrinsic<[llvm_anyvector_ty],
1124                 [LLVMMatchType<0>,
1125                  LLVMVectorOfBitcastsToInt<0>],
1126                 [IntrNoMem]>;
1127
1128   class AdvSIMD_SVE2_TBX_Intrinsic
1129     : Intrinsic<[llvm_anyvector_ty],
1130                 [LLVMMatchType<0>,
1131                  LLVMMatchType<0>,
1132                  LLVMVectorOfBitcastsToInt<0>],
1133                 [IntrNoMem]>;
1134
1135   class SVE2_1VectorArg_Long_Intrinsic
1136     : Intrinsic<[llvm_anyvector_ty],
1137                 [LLVMSubdivide2VectorType<0>,
1138                  llvm_i32_ty],
1139                 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1140
1141   class SVE2_2VectorArg_Long_Intrinsic
1142     : Intrinsic<[llvm_anyvector_ty],
1143                 [LLVMSubdivide2VectorType<0>,
1144                  LLVMSubdivide2VectorType<0>],
1145                 [IntrNoMem]>;
1146
1147   class SVE2_2VectorArgIndexed_Long_Intrinsic
1148   : Intrinsic<[llvm_anyvector_ty],
1149               [LLVMSubdivide2VectorType<0>,
1150                LLVMSubdivide2VectorType<0>,
1151                llvm_i32_ty],
1152               [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1153
1154   class SVE2_2VectorArg_Wide_Intrinsic
1155     : Intrinsic<[llvm_anyvector_ty],
1156                 [LLVMMatchType<0>,
1157                  LLVMSubdivide2VectorType<0>],
1158                 [IntrNoMem]>;
1159
1160   class SVE2_2VectorArg_Pred_Long_Intrinsic
1161     : Intrinsic<[llvm_anyvector_ty],
1162                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1163                  LLVMMatchType<0>,
1164                  LLVMSubdivide2VectorType<0>],
1165                 [IntrNoMem]>;
1166
1167   class SVE2_3VectorArg_Long_Intrinsic
1168     : Intrinsic<[llvm_anyvector_ty],
1169                 [LLVMMatchType<0>,
1170                  LLVMSubdivide2VectorType<0>,
1171                  LLVMSubdivide2VectorType<0>],
1172                 [IntrNoMem]>;
1173
1174   class SVE2_3VectorArgIndexed_Long_Intrinsic
1175     : Intrinsic<[llvm_anyvector_ty],
1176                 [LLVMMatchType<0>,
1177                  LLVMSubdivide2VectorType<0>,
1178                  LLVMSubdivide2VectorType<0>,
1179                  llvm_i32_ty],
1180                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1181
1182   class SVE2_1VectorArg_Narrowing_Intrinsic
1183     : Intrinsic<[LLVMSubdivide2VectorType<0>],
1184                 [llvm_anyvector_ty],
1185                 [IntrNoMem]>;
1186
1187   class SVE2_Merged1VectorArg_Narrowing_Intrinsic
1188     : Intrinsic<[LLVMSubdivide2VectorType<0>],
1189                 [LLVMSubdivide2VectorType<0>,
1190                  llvm_anyvector_ty],
1191                 [IntrNoMem]>;
1192   class SVE2_2VectorArg_Narrowing_Intrinsic
1193       : Intrinsic<
1194             [LLVMSubdivide2VectorType<0>],
1195             [llvm_anyvector_ty, LLVMMatchType<0>],
1196             [IntrNoMem]>;
1197
1198   class SVE2_Merged2VectorArg_Narrowing_Intrinsic
1199       : Intrinsic<
1200             [LLVMSubdivide2VectorType<0>],
1201             [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
1202             [IntrNoMem]>;
1203
1204   class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
1205       : Intrinsic<[LLVMSubdivide2VectorType<0>],
1206                   [llvm_anyvector_ty, llvm_i32_ty],
1207                   [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1208
1209   class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
1210       : Intrinsic<[LLVMSubdivide2VectorType<0>],
1211                   [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
1212                    llvm_i32_ty],
1213                   [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1214
1215   class SVE2_CONFLICT_DETECT_Intrinsic
1216     : Intrinsic<[llvm_anyvector_ty],
1217                 [LLVMAnyPointerType<llvm_any_ty>,
1218                  LLVMMatchType<1>]>;
1219
1220   class SVE2_3VectorArg_Indexed_Intrinsic
1221     : Intrinsic<[llvm_anyvector_ty],
1222                 [LLVMMatchType<0>,
1223                  LLVMSubdivide2VectorType<0>,
1224                  LLVMSubdivide2VectorType<0>,
1225                  llvm_i32_ty],
1226                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1227
1228   class AdvSIMD_SVE_CDOT_LANE_Intrinsic
1229     : Intrinsic<[llvm_anyvector_ty],
1230                 [LLVMMatchType<0>,
1231                  LLVMSubdivide4VectorType<0>,
1232                  LLVMSubdivide4VectorType<0>,
1233                  llvm_i32_ty,
1234                  llvm_i32_ty],
1235                 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1236
1237   // NOTE: There is no relationship between these intrinsics beyond an attempt
1238   // to reuse currently identical class definitions.
1239   class AdvSIMD_SVE_LOGB_Intrinsic  : AdvSIMD_SVE_CNT_Intrinsic;
1240   class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
1241   class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;
1242
1243   // This class of intrinsics are not intended to be useful within LLVM IR but
1244   // are instead here to support some of the more regid parts of the ACLE.
1245   class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN>
1246       : Intrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
1247 }
1248
1249 //===----------------------------------------------------------------------===//
1250 // SVE
1251
1252 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
1253
1254 class AdvSIMD_SVE_Reduce_Intrinsic
1255   : Intrinsic<[LLVMVectorElementType<0>],
1256               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1257                llvm_anyvector_ty],
1258               [IntrNoMem]>;
1259
1260 class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
1261   : Intrinsic<[llvm_i64_ty],
1262               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1263                llvm_anyvector_ty],
1264               [IntrNoMem]>;
1265
1266 class AdvSIMD_SVE_WHILE_Intrinsic
1267     : Intrinsic<[llvm_anyvector_ty],
1268                 [llvm_anyint_ty, LLVMMatchType<1>],
1269                 [IntrNoMem]>;
1270
1271 class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
1272     : Intrinsic<[llvm_anyvector_ty],
1273                 [
1274                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1275                   LLVMPointerToElt<0>,
1276                   LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1277                 ],
1278                 [IntrReadMem, IntrArgMemOnly]>;
1279
1280 class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
1281     : Intrinsic<[llvm_anyvector_ty],
1282                 [
1283                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1284                   LLVMPointerToElt<0>,
1285                   LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1286                 ],
1287                 [IntrReadMem, IntrArgMemOnly]>;
1288
1289 class AdvSIMD_GatherLoad_VS_Intrinsic
1290     : Intrinsic<[llvm_anyvector_ty],
1291                 [
1292                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1293                   llvm_anyvector_ty,
1294                   llvm_i64_ty
1295                 ],
1296                 [IntrReadMem, IntrArgMemOnly]>;
1297
1298 class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
1299     : Intrinsic<[],
1300                [
1301                  llvm_anyvector_ty,
1302                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1303                  LLVMPointerToElt<0>,
1304                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1305                ],
1306                [IntrWriteMem, IntrArgMemOnly]>;
1307
1308 class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
1309     : Intrinsic<[],
1310                [
1311                  llvm_anyvector_ty,
1312                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1313                  LLVMPointerToElt<0>,
1314                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1315                ],
1316                [IntrWriteMem, IntrArgMemOnly]>;
1317
1318 class AdvSIMD_ScatterStore_VS_Intrinsic
1319     : Intrinsic<[],
1320                [
1321                  llvm_anyvector_ty,
1322                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1323                  llvm_anyvector_ty, llvm_i64_ty
1324                ],
1325                [IntrWriteMem, IntrArgMemOnly]>;
1326
1327
1328 class SVE_gather_prf_SV
1329     : Intrinsic<[],
1330                 [
1331                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1332                   llvm_ptr_ty, // Base address
1333                   llvm_anyvector_ty, // Offsets
1334                   llvm_i32_ty // Prfop
1335                 ],
1336                 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;
1337
1338 class SVE_gather_prf_VS
1339     : Intrinsic<[],
1340                 [
1341                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1342                   llvm_anyvector_ty, // Base addresses
1343                   llvm_i64_ty, // Scalar offset
1344                   llvm_i32_ty // Prfop
1345                 ],
1346                 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;
1347
1348 class SVE_MatMul_Intrinsic
1349     : Intrinsic<[llvm_anyvector_ty],
1350                 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
1351                 [IntrNoMem]>;
1352
1353 class SVE_4Vec_BF16
1354     : Intrinsic<[llvm_nxv4f32_ty],
1355                 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
1356                 [IntrNoMem]>;
1357
1358 class SVE_4Vec_BF16_Indexed
1359     : Intrinsic<[llvm_nxv4f32_ty],
1360                 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
1361                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1362
1363 //
1364 // Vector tuple creation intrinsics (ACLE)
1365 //
1366
1367 def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
1368 def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
1369 def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;
1370
1371 //
1372 // Vector tuple insertion/extraction intrinsics (ACLE)
1373 //
1374
1375 def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
1376 def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;
1377
1378 //
1379 // Loads
1380 //
1381
1382 def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;
1383
1384 def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1385 def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1386 def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1387
1388 def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1389 def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1390 def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1391
1392 def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1393 def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
1394
1395 //
1396 // Stores
1397 //
1398
1399 def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
1400 def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
1401 def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
1402 def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;
1403
1404 def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1405
1406 //
1407 // Prefetches
1408 //
1409
1410 def int_aarch64_sve_prf
1411   : Intrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
1412                   [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
1413
1414 // Scalar + 32-bit scaled offset vector, zero extend, packed and
1415 // unpacked.
1416 def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
1417 def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
1418 def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
1419 def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;
1420
1421 // Scalar + 32-bit scaled offset vector, sign extend, packed and
1422 // unpacked.
1423 def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
1424 def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
1425 def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
1426 def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;
1427
1428 // Scalar + 64-bit scaled offset vector.
1429 def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
1430 def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
1431 def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
1432 def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;
1433
1434 // Vector + scalar.
1435 def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
1436 def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
1437 def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
1438 def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;
1439
1440 //
1441 // Scalar to vector operations
1442 //
1443
1444 def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
1445 def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;
1446
1447
1448 def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;
1449
1450 //
1451 // Address calculation
1452 //
1453
1454 def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
1455 def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
1456 def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
1457 def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;
1458
1459 //
1460 // Integer arithmetic
1461 //
1462
1463 def int_aarch64_sve_add   : AdvSIMD_Pred2VectorArg_Intrinsic;
1464 def int_aarch64_sve_sub   : AdvSIMD_Pred2VectorArg_Intrinsic;
1465 def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1466
1467 def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;
1468
1469 def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
1470 def int_aarch64_sve_mul_lane   : AdvSIMD_2VectorArgIndexed_Intrinsic;
1471 def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1472 def int_aarch64_sve_umulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1473
1474 def int_aarch64_sve_sdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1475 def int_aarch64_sve_udiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1476 def int_aarch64_sve_sdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1477 def int_aarch64_sve_udivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1478
1479 def int_aarch64_sve_smax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1480 def int_aarch64_sve_umax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1481 def int_aarch64_sve_smin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1482 def int_aarch64_sve_umin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1483 def int_aarch64_sve_sabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1484 def int_aarch64_sve_uabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1485
1486 def int_aarch64_sve_mad        : AdvSIMD_Pred3VectorArg_Intrinsic;
1487 def int_aarch64_sve_msb        : AdvSIMD_Pred3VectorArg_Intrinsic;
1488 def int_aarch64_sve_mla        : AdvSIMD_Pred3VectorArg_Intrinsic;
1489 def int_aarch64_sve_mla_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1490 def int_aarch64_sve_mls        : AdvSIMD_Pred3VectorArg_Intrinsic;
1491 def int_aarch64_sve_mls_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1492
1493 def int_aarch64_sve_saddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1494 def int_aarch64_sve_uaddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1495
1496 def int_aarch64_sve_smaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1497 def int_aarch64_sve_umaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1498 def int_aarch64_sve_sminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1499 def int_aarch64_sve_uminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1500
1501 def int_aarch64_sve_orv        : AdvSIMD_SVE_Reduce_Intrinsic;
1502 def int_aarch64_sve_eorv       : AdvSIMD_SVE_Reduce_Intrinsic;
1503 def int_aarch64_sve_andv       : AdvSIMD_SVE_Reduce_Intrinsic;
1504
1505 def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
1506 def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
1507
1508 def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
1509 def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1510
1511 def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
1512 def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1513
1514 def int_aarch64_sve_sqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1515 def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1516 def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1517 def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1518
1519 // Shifts
1520
1521 def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1522 def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1523 def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
1524 def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
1525 def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
1526 def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1527 def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1528 def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1529
1530 //
1531 // Integer comparisons
1532 //
1533
1534 def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1535 def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
1536 def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1537 def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
1538 def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
1539 def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
1540
1541 def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1542 def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1543 def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1544 def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1545 def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1546 def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1547 def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1548 def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1549 def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1550 def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1551
1552 //
1553 // Counting bits
1554 //
1555
1556 def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
1557 def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
1558 def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
1559
1560 //
1561 // Counting elements
1562 //
1563
1564 def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
1565 def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
1566 def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
1567 def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
1568
1569 def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
1570
1571 //
1572 // FFR manipulation
1573 //
1574
1575 def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   Intrinsic<[llvm_nxv16i1_ty], []>;
1576 def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
1577 def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  Intrinsic<[], []>;
1578 def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   Intrinsic<[], [llvm_nxv16i1_ty]>;
1579
1580 //
1581 // Saturating scalar arithmetic
1582 //
1583
1584 def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1585 def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1586 def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1587 def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1588
1589 def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1590 def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1591 def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1592 def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1593 def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1594 def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1595 def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1596 def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1597 def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1598 def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1599
1600 def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1601 def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1602 def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1603 def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1604
1605 def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1606 def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1607 def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1608 def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1609 def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1610 def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1611 def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1612 def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1613 def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1614 def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1615
1616 def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1617 def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1618 def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1619 def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1620
1621 def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1622 def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1623 def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1624 def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1625 def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1626 def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1627 def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1628 def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1629 def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1630 def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1631
1632 def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1633 def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1634 def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1635 def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1636
1637 def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1638 def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1639 def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1640 def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1641 def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1642 def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1643 def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1644 def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1645 def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1646 def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1647
1648 //
1649 // Reversal
1650 //
1651
1652 def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
1653 def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
1654 def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
1655 def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;
1656
1657 //
1658 // Permutations and selection
1659 //
1660
1661 def int_aarch64_sve_clasta    : AdvSIMD_Pred2VectorArg_Intrinsic;
1662 def int_aarch64_sve_clasta_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1663 def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
1664 def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1665 def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
1666 def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
1667 def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
1668 def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
1669 def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
1670 def int_aarch64_sve_lastb     : AdvSIMD_SVE_Reduce_Intrinsic;
1671 def int_aarch64_sve_rev       : AdvSIMD_1VectorArg_Intrinsic;
1672 def int_aarch64_sve_splice    : AdvSIMD_Pred2VectorArg_Intrinsic;
1673 def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1674 def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1675 def int_aarch64_sve_tbl       : AdvSIMD_SVE_TBL_Intrinsic;
1676 def int_aarch64_sve_trn1      : AdvSIMD_2VectorArg_Intrinsic;
1677 def int_aarch64_sve_trn2      : AdvSIMD_2VectorArg_Intrinsic;
1678 def int_aarch64_sve_trn1q     : AdvSIMD_2VectorArg_Intrinsic;
1679 def int_aarch64_sve_trn2q     : AdvSIMD_2VectorArg_Intrinsic;
1680 def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1681 def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1682 def int_aarch64_sve_uzp1      : AdvSIMD_2VectorArg_Intrinsic;
1683 def int_aarch64_sve_uzp2      : AdvSIMD_2VectorArg_Intrinsic;
1684 def int_aarch64_sve_uzp1q     : AdvSIMD_2VectorArg_Intrinsic;
1685 def int_aarch64_sve_uzp2q     : AdvSIMD_2VectorArg_Intrinsic;
1686 def int_aarch64_sve_zip1      : AdvSIMD_2VectorArg_Intrinsic;
1687 def int_aarch64_sve_zip2      : AdvSIMD_2VectorArg_Intrinsic;
1688 def int_aarch64_sve_zip1q     : AdvSIMD_2VectorArg_Intrinsic;
1689 def int_aarch64_sve_zip2q     : AdvSIMD_2VectorArg_Intrinsic;
1690
1691 //
1692 // Logical operations
1693 //
1694
1695 def int_aarch64_sve_and  : AdvSIMD_Pred2VectorArg_Intrinsic;
1696 def int_aarch64_sve_bic  : AdvSIMD_Pred2VectorArg_Intrinsic;
1697 def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
1698 def int_aarch64_sve_eor  : AdvSIMD_Pred2VectorArg_Intrinsic;
1699 def int_aarch64_sve_not  : AdvSIMD_Merged1VectorArg_Intrinsic;
1700 def int_aarch64_sve_orr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1701
1702 //
1703 // Conversion
1704 //
1705
1706 def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1707 def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1708 def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1709 def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1710 def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1711 def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1712
1713 //
1714 // While comparisons
1715 //
1716
1717 def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
1718 def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
1719 def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
1720 def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
1721 def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
1722 def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
1723 def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
1724 def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;
1725
1726 //
1727 // Floating-point arithmetic
1728 //
1729
1730 def int_aarch64_sve_fabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1731 def int_aarch64_sve_fabs       : AdvSIMD_Merged1VectorArg_Intrinsic;
1732 def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1733 def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
1734 def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
1735 def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
1736 def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1737 def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1738 def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
1739 def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
1740 def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1741 def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1742 def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1743 def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1744 def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
1745 def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1746 def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
1747 def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1748 def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;
1749 def int_aarch64_sve_fmul       : AdvSIMD_Pred2VectorArg_Intrinsic;
1750 def int_aarch64_sve_fmulx      : AdvSIMD_Pred2VectorArg_Intrinsic;
1751 def int_aarch64_sve_fneg       : AdvSIMD_Merged1VectorArg_Intrinsic;
1752 def int_aarch64_sve_fmul_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
1753 def int_aarch64_sve_fnmad      : AdvSIMD_Pred3VectorArg_Intrinsic;
1754 def int_aarch64_sve_fnmla      : AdvSIMD_Pred3VectorArg_Intrinsic;
1755 def int_aarch64_sve_fnmls      : AdvSIMD_Pred3VectorArg_Intrinsic;
1756 def int_aarch64_sve_fnmsb      : AdvSIMD_Pred3VectorArg_Intrinsic;
1757 def int_aarch64_sve_frecpe_x   : AdvSIMD_1VectorArg_Intrinsic;
1758 def int_aarch64_sve_frecps_x   : AdvSIMD_2VectorArg_Intrinsic;
1759 def int_aarch64_sve_frecpx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1760 def int_aarch64_sve_frinta     : AdvSIMD_Merged1VectorArg_Intrinsic;
1761 def int_aarch64_sve_frinti     : AdvSIMD_Merged1VectorArg_Intrinsic;
1762 def int_aarch64_sve_frintm     : AdvSIMD_Merged1VectorArg_Intrinsic;
1763 def int_aarch64_sve_frintn     : AdvSIMD_Merged1VectorArg_Intrinsic;
1764 def int_aarch64_sve_frintp     : AdvSIMD_Merged1VectorArg_Intrinsic;
1765 def int_aarch64_sve_frintx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1766 def int_aarch64_sve_frintz     : AdvSIMD_Merged1VectorArg_Intrinsic;
1767 def int_aarch64_sve_frsqrte_x  : AdvSIMD_1VectorArg_Intrinsic;
1768 def int_aarch64_sve_frsqrts_x  : AdvSIMD_2VectorArg_Intrinsic;
1769 def int_aarch64_sve_fscale     : AdvSIMD_SVE_SCALE_Intrinsic;
1770 def int_aarch64_sve_fsqrt      : AdvSIMD_Merged1VectorArg_Intrinsic;
1771 def int_aarch64_sve_fsub       : AdvSIMD_Pred2VectorArg_Intrinsic;
1772 def int_aarch64_sve_fsubr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1773 def int_aarch64_sve_ftmad_x    : AdvSIMD_2VectorArgIndexed_Intrinsic;
1774 def int_aarch64_sve_ftsmul_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1775 def int_aarch64_sve_ftssel_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1776
1777 //
1778 // Floating-point reductions
1779 //
1780
1781 def int_aarch64_sve_fadda   : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1782 def int_aarch64_sve_faddv   : AdvSIMD_SVE_Reduce_Intrinsic;
1783 def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
1784 def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1785 def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
1786 def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1787
1788 //
1789 // Floating-point conversions
1790 //
1791
1792 def int_aarch64_sve_fcvt   : AdvSIMD_SVE_FCVT_Intrinsic;
1793 def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
1794 def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
1795 def int_aarch64_sve_scvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1796 def int_aarch64_sve_ucvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1797
1798 //
1799 // Floating-point comparisons
1800 //
1801
1802 def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
1803 def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;
1804
1805 def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1806 def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
1807 def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1808 def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
1809 def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;
1810
1811 def int_aarch64_sve_fcvtzs_i32f16   : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1812 def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1813 def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1814 def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1815
1816 def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<"svcvt_bf16_f32_m",   llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1817 def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1818
1819 def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1820 def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1821 def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1822 def int_aarch64_sve_fcvtzu_i64f32   : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1823
1824 def int_aarch64_sve_fcvt_f16f32     : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1825 def int_aarch64_sve_fcvt_f16f64     : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1826 def int_aarch64_sve_fcvt_f32f64     : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1827
1828 def int_aarch64_sve_fcvt_f32f16     : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1829 def int_aarch64_sve_fcvt_f64f16     : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1830 def int_aarch64_sve_fcvt_f64f32     : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1831
1832 def int_aarch64_sve_fcvtlt_f32f16   : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1833 def int_aarch64_sve_fcvtlt_f64f32   : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1834 def int_aarch64_sve_fcvtnt_f16f32   : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1835 def int_aarch64_sve_fcvtnt_f32f64   : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1836
1837 def int_aarch64_sve_fcvtx_f32f64    : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1838 def int_aarch64_sve_fcvtxnt_f32f64  : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1839
1840 def int_aarch64_sve_scvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1841 def int_aarch64_sve_scvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1842 def int_aarch64_sve_scvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1843 def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1844
1845 def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1846 def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1847 def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1848 def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1849
1850 //
1851 // Predicate creation
1852 //
1853
1854 def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
1855
1856 //
1857 // Predicate operations
1858 //
1859
1860 def int_aarch64_sve_and_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1861 def int_aarch64_sve_bic_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1862 def int_aarch64_sve_brka    : AdvSIMD_Merged1VectorArg_Intrinsic;
1863 def int_aarch64_sve_brka_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1864 def int_aarch64_sve_brkb    : AdvSIMD_Merged1VectorArg_Intrinsic;
1865 def int_aarch64_sve_brkb_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1866 def int_aarch64_sve_brkn_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1867 def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1868 def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1869 def int_aarch64_sve_eor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1870 def int_aarch64_sve_nand_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1871 def int_aarch64_sve_nor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1872 def int_aarch64_sve_orn_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1873 def int_aarch64_sve_orr_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1874 def int_aarch64_sve_pfirst  : AdvSIMD_Pred1VectorArg_Intrinsic;
1875 def int_aarch64_sve_pnext   : AdvSIMD_Pred1VectorArg_Intrinsic;
1876 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1877 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1878
1879 //
1880 // Testing predicates
1881 //
1882
1883 def int_aarch64_sve_ptest_any   : AdvSIMD_SVE_PTEST_Intrinsic;
1884 def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
1885 def int_aarch64_sve_ptest_last  : AdvSIMD_SVE_PTEST_Intrinsic;
1886
1887 //
1888 // Reinterpreting data
1889 //
1890
1891 def int_aarch64_sve_convert_from_svbool : Intrinsic<[llvm_anyvector_ty],
1892                                                     [llvm_nxv16i1_ty],
1893                                                     [IntrNoMem]>;
1894
1895 def int_aarch64_sve_convert_to_svbool : Intrinsic<[llvm_nxv16i1_ty],
1896                                                   [llvm_anyvector_ty],
1897                                                   [IntrNoMem]>;
1898
1899 //
1900 // Gather loads: scalar base + vector offsets
1901 //
1902
1903 // 64 bit unscaled offsets
1904 def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1905
1906 // 64 bit scaled offsets
1907 def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1908
1909 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1910 def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1911 def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1912
1913 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1914 def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1915 def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1916
1917 //
1918 // Gather loads: vector base + scalar offset
1919 //
1920
1921 def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1922
1923
1924 //
1925 // First-faulting gather loads: scalar base + vector offsets
1926 //
1927
1928 // 64 bit unscaled offsets
1929 def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1930
1931 // 64 bit scaled offsets
1932 def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1933
1934 // 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1935 def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1936 def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1937
1938 // 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1939 def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1940 def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1941
1942 //
1943 // First-faulting gather loads: vector base + scalar offset
1944 //
1945
1946 def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1947
1948
1949 //
1950 // Non-temporal gather loads: scalar base + vector offsets
1951 //
1952
1953 // 64 bit unscaled offsets
1954 def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1955
1956 // 64 bit indices
1957 def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1958
1959 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
1960 def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1961
1962 //
1963 // Non-temporal gather loads: vector base + scalar offset
1964 //
1965
1966 def int_aarch64_sve_ldnt1_gather_scalar_offset  : AdvSIMD_GatherLoad_VS_Intrinsic;
1967
1968 //
1969 // Scatter stores: scalar base + vector offsets
1970 //
1971
1972 // 64 bit unscaled offsets
1973 def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
1974
1975 // 64 bit scaled offsets
1976 def int_aarch64_sve_st1_scatter_index
1977     : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
1978
1979 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1980 def int_aarch64_sve_st1_scatter_sxtw
1981     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1982
1983 def int_aarch64_sve_st1_scatter_uxtw
1984     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1985
1986 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1987 def int_aarch64_sve_st1_scatter_sxtw_index
1988     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1989
1990 def int_aarch64_sve_st1_scatter_uxtw_index
1991     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1992
1993 //
1994 // Scatter stores: vector base + scalar offset
1995 //
1996
1997 def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
1998
1999 //
2000 // Non-temporal scatter stores: scalar base + vector offsets
2001 //
2002
2003 // 64 bit unscaled offsets
2004 def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2005
2006 // 64 bit indices
2007 def int_aarch64_sve_stnt1_scatter_index
2008     : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2009
2010 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2011 def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2012
2013 //
2014 // Non-temporal scatter stores: vector base + scalar offset
2015 //
2016
2017 def int_aarch64_sve_stnt1_scatter_scalar_offset  : AdvSIMD_ScatterStore_VS_Intrinsic;
2018
2019 //
2020 // SVE2 - Uniform DSP operations
2021 //
2022
2023 def int_aarch64_sve_saba          : AdvSIMD_3VectorArg_Intrinsic;
2024 def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2025 def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2026 def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2027 def int_aarch64_sve_sli           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2028 def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
2029 def int_aarch64_sve_sqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2030 def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
2031 def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
2032 def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
2033 def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
2034 def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2035 def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
2036 def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2037 def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
2038 def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2039 def int_aarch64_sve_sqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2040 def int_aarch64_sve_sqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2041 def int_aarch64_sve_sqshlu        : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2042 def int_aarch64_sve_sqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2043 def int_aarch64_sve_sqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2044 def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2045 def int_aarch64_sve_sri           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2046 def int_aarch64_sve_srshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2047 def int_aarch64_sve_srshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2048 def int_aarch64_sve_srsra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2049 def int_aarch64_sve_ssra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2050 def int_aarch64_sve_suqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2051 def int_aarch64_sve_uaba          : AdvSIMD_3VectorArg_Intrinsic;
2052 def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2053 def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2054 def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2055 def int_aarch64_sve_uqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2056 def int_aarch64_sve_uqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2057 def int_aarch64_sve_uqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2058 def int_aarch64_sve_uqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2059 def int_aarch64_sve_uqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2060 def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
2061 def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2062 def int_aarch64_sve_urshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2063 def int_aarch64_sve_urshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2064 def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;
2065 def int_aarch64_sve_ursra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2066 def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2067 def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2068
2069 //
2070 // SVE2 - Widening DSP operations
2071 //
2072
2073 def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
2074 def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
2075 def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
2076 def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
2077 def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
2078 def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
2079 def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
2080 def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
2081 def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
2082 def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
2083 def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
2084 def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
2085 def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
2086 def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
2087 def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
2088 def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
2089 def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
2090 def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
2091 def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
2092 def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
2093 def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
2094 def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
2095 def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
2096 def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
2097 def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
2098 def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
2099 def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
2100 def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;
2101
2102 //
2103 // SVE2 - Non-widening pairwise arithmetic
2104 //
2105
2106 def int_aarch64_sve_addp    : AdvSIMD_Pred2VectorArg_Intrinsic;
2107 def int_aarch64_sve_faddp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2108 def int_aarch64_sve_fmaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2109 def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2110 def int_aarch64_sve_fminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2111 def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2112 def int_aarch64_sve_smaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2113 def int_aarch64_sve_sminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2114 def int_aarch64_sve_umaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2115 def int_aarch64_sve_uminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2116
2117 //
2118 // SVE2 - Widening pairwise arithmetic
2119 //
2120
2121 def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2122 def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2123
2124 //
2125 // SVE2 - Uniform complex integer arithmetic
2126 //
2127
2128 def int_aarch64_sve_cadd_x           : AdvSIMD_SVE2_CADD_Intrinsic;
2129 def int_aarch64_sve_sqcadd_x         : AdvSIMD_SVE2_CADD_Intrinsic;
2130 def int_aarch64_sve_cmla_x           : AdvSIMD_SVE2_CMLA_Intrinsic;
2131 def int_aarch64_sve_cmla_lane_x      : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2132 def int_aarch64_sve_sqrdcmlah_x      : AdvSIMD_SVE2_CMLA_Intrinsic;
2133 def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2134
2135 //
2136 // SVE2 - Widening complex integer arithmetic
2137 //
2138
2139 def int_aarch64_sve_saddlbt   : SVE2_2VectorArg_Long_Intrinsic;
2140 def int_aarch64_sve_ssublbt   : SVE2_2VectorArg_Long_Intrinsic;
2141 def int_aarch64_sve_ssubltb   : SVE2_2VectorArg_Long_Intrinsic;
2142
2143 //
2144 // SVE2 - Widening complex integer dot product
2145 //
2146
2147 def int_aarch64_sve_cdot      : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2148 def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;
2149
2150 //
2151 // SVE2 - Floating-point widening multiply-accumulate
2152 //
2153
2154 def int_aarch64_sve_fmlalb        : SVE2_3VectorArg_Long_Intrinsic;
2155 def int_aarch64_sve_fmlalb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2156 def int_aarch64_sve_fmlalt        : SVE2_3VectorArg_Long_Intrinsic;
2157 def int_aarch64_sve_fmlalt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2158 def int_aarch64_sve_fmlslb        : SVE2_3VectorArg_Long_Intrinsic;
2159 def int_aarch64_sve_fmlslb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2160 def int_aarch64_sve_fmlslt        : SVE2_3VectorArg_Long_Intrinsic;
2161 def int_aarch64_sve_fmlslt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2162
2163 //
2164 // SVE2 - Floating-point integer binary logarithm
2165 //
2166
2167 def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;
2168
2169 //
2170 // SVE2 - Vector histogram count
2171 //
2172
2173 def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
2174 def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;
2175
2176 //
2177 // SVE2 - Character match
2178 //
2179
2180 def int_aarch64_sve_match   : AdvSIMD_SVE_Compare_Intrinsic;
2181 def int_aarch64_sve_nmatch  : AdvSIMD_SVE_Compare_Intrinsic;
2182
2183 //
2184 // SVE2 - Unary narrowing operations
2185 //
2186
2187 def int_aarch64_sve_sqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2188 def int_aarch64_sve_sqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2189 def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
2190 def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2191 def int_aarch64_sve_uqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2192 def int_aarch64_sve_uqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2193
2194 //
2195 // SVE2 - Binary narrowing DSP operations
2196 //
2197 def int_aarch64_sve_addhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2198 def int_aarch64_sve_addhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2199
2200 def int_aarch64_sve_raddhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2201 def int_aarch64_sve_raddhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2202
2203 def int_aarch64_sve_subhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2204 def int_aarch64_sve_subhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2205
2206 def int_aarch64_sve_rsubhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2207 def int_aarch64_sve_rsubhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2208
2209 // Narrowing shift right
2210 def int_aarch64_sve_shrnb     : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2211 def int_aarch64_sve_shrnt     : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2212
2213 def int_aarch64_sve_rshrnb    : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2214 def int_aarch64_sve_rshrnt    : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2215
2216 // Saturating shift right - signed input/output
2217 def int_aarch64_sve_sqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2218 def int_aarch64_sve_sqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2219
2220 def int_aarch64_sve_sqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2221 def int_aarch64_sve_sqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2222
2223 // Saturating shift right - unsigned input/output
2224 def int_aarch64_sve_uqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2225 def int_aarch64_sve_uqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2226
2227 def int_aarch64_sve_uqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2228 def int_aarch64_sve_uqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2229
2230 // Saturating shift right - signed input, unsigned output
2231 def int_aarch64_sve_sqshrunb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2232 def int_aarch64_sve_sqshrunt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2233
2234 def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2235 def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2236
2237 // SVE2 MLA LANE.
2238 def int_aarch64_sve_smlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2239 def int_aarch64_sve_smlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2240 def int_aarch64_sve_umlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2241 def int_aarch64_sve_umlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2242 def int_aarch64_sve_smlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2243 def int_aarch64_sve_smlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2244 def int_aarch64_sve_umlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2245 def int_aarch64_sve_umlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2246 def int_aarch64_sve_smullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2247 def int_aarch64_sve_smullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2248 def int_aarch64_sve_umullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2249 def int_aarch64_sve_umullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2250 def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2251 def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2252 def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2253 def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2254 def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2255 def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2256
2257 // SVE2 MLA Unpredicated.
2258 def int_aarch64_sve_smlalb      : SVE2_3VectorArg_Long_Intrinsic;
2259 def int_aarch64_sve_smlalt      : SVE2_3VectorArg_Long_Intrinsic;
2260 def int_aarch64_sve_umlalb      : SVE2_3VectorArg_Long_Intrinsic;
2261 def int_aarch64_sve_umlalt      : SVE2_3VectorArg_Long_Intrinsic;
2262 def int_aarch64_sve_smlslb      : SVE2_3VectorArg_Long_Intrinsic;
2263 def int_aarch64_sve_smlslt      : SVE2_3VectorArg_Long_Intrinsic;
2264 def int_aarch64_sve_umlslb      : SVE2_3VectorArg_Long_Intrinsic;
2265 def int_aarch64_sve_umlslt      : SVE2_3VectorArg_Long_Intrinsic;
2266 def int_aarch64_sve_smullb      : SVE2_2VectorArg_Long_Intrinsic;
2267 def int_aarch64_sve_smullt      : SVE2_2VectorArg_Long_Intrinsic;
2268 def int_aarch64_sve_umullb      : SVE2_2VectorArg_Long_Intrinsic;
2269 def int_aarch64_sve_umullt      : SVE2_2VectorArg_Long_Intrinsic;
2270
2271 def int_aarch64_sve_sqdmlalb    : SVE2_3VectorArg_Long_Intrinsic;
2272 def int_aarch64_sve_sqdmlalt    : SVE2_3VectorArg_Long_Intrinsic;
2273 def int_aarch64_sve_sqdmlslb    : SVE2_3VectorArg_Long_Intrinsic;
2274 def int_aarch64_sve_sqdmlslt    : SVE2_3VectorArg_Long_Intrinsic;
2275 def int_aarch64_sve_sqdmullb    : SVE2_2VectorArg_Long_Intrinsic;
2276 def int_aarch64_sve_sqdmullt    : SVE2_2VectorArg_Long_Intrinsic;
2277 def int_aarch64_sve_sqdmlalbt   : SVE2_3VectorArg_Long_Intrinsic;
2278 def int_aarch64_sve_sqdmlslbt   : SVE2_3VectorArg_Long_Intrinsic;
2279
2280 // SVE2 ADDSUB Long Unpredicated.
2281 def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
2282 def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
2283 def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
2284 def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;
2285
2286 //
2287 // SVE2 - Polynomial arithmetic
2288 //
2289 def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
2290 def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
2291 def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
2292 def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
2293
2294 //
2295 // SVE2 bitwise ternary operations.
2296 //
2297 def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
2298 def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
2299 def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
2300 def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
2301 def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
2302 def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
2303 def int_aarch64_sve_xar    : AdvSIMD_2VectorArgIndexed_Intrinsic;
2304
2305 //
2306 // SVE2 - Optional AES, SHA-3 and SM4
2307 //
2308
2309 def int_aarch64_sve_aesd    : GCCBuiltin<"__builtin_sve_svaesd_u8">,
2310                               Intrinsic<[llvm_nxv16i8_ty],
2311                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2312                                         [IntrNoMem]>;
2313 def int_aarch64_sve_aesimc  : GCCBuiltin<"__builtin_sve_svaesimc_u8">,
2314                               Intrinsic<[llvm_nxv16i8_ty],
2315                                         [llvm_nxv16i8_ty],
2316                                         [IntrNoMem]>;
2317 def int_aarch64_sve_aese    : GCCBuiltin<"__builtin_sve_svaese_u8">,
2318                               Intrinsic<[llvm_nxv16i8_ty],
2319                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2320                                         [IntrNoMem]>;
2321 def int_aarch64_sve_aesmc   : GCCBuiltin<"__builtin_sve_svaesmc_u8">,
2322                               Intrinsic<[llvm_nxv16i8_ty],
2323                                         [llvm_nxv16i8_ty],
2324                                         [IntrNoMem]>;
2325 def int_aarch64_sve_rax1    : GCCBuiltin<"__builtin_sve_svrax1_u64">,
2326                               Intrinsic<[llvm_nxv2i64_ty],
2327                                         [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
2328                                         [IntrNoMem]>;
2329 def int_aarch64_sve_sm4e    : GCCBuiltin<"__builtin_sve_svsm4e_u32">,
2330                               Intrinsic<[llvm_nxv4i32_ty],
2331                                         [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2332                                         [IntrNoMem]>;
2333 def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">,
2334                               Intrinsic<[llvm_nxv4i32_ty],
2335                                         [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2336                                         [IntrNoMem]>;
2337 //
2338 // SVE2 - Extended table lookup/permute
2339 //
2340
2341 def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
2342 def int_aarch64_sve_tbx  : AdvSIMD_SVE2_TBX_Intrinsic;
2343
2344 //
2345 // SVE2 - Optional bit permutation
2346 //
2347
2348 def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
2349 def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
2350 def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;
2351
2352
2353 //
2354 // SVE ACLE: 7.3. INT8 matrix multiply extensions
2355 //
2356 def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
2357 def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
2358 def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;
2359
2360 def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
2361 def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2362 def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2363
2364 //
2365 // SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
2366 //
2367 def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
2368
2369 //
2370 // SVE ACLE: 7.2. BFloat16 extensions
2371 //
2372
2373 def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
2374 def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
2375 def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
2376
2377 def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;
2378
2379 def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
2380 def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
2381 def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
2382 }
2383
2384 //
2385 // SVE2 - Contiguous conflict detection
2386 //
2387
2388 def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2389 def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2390 def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2391 def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2392 def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2393 def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2394 def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2395 def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;