contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsAArch64.td

   1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines all of the AARCH64-specific intrinsics.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 let TargetPrefix = "aarch64" in {
  14
  15 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
  16                                  [IntrNoFree, IntrWillReturn]>;
  17 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
  18                                   [IntrNoFree, IntrWillReturn]>;
  19 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
  20                                  [IntrNoFree, IntrWillReturn]>;
  21 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
  22                                   [IntrNoFree, IntrWillReturn]>;
  23
  24 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
  25                                  [IntrNoFree, IntrWillReturn]>;
  26 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
  27                                   [IntrNoFree, IntrWillReturn]>;
  28 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
  29                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
  30                                [IntrNoFree, IntrWillReturn]>;
  31 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
  32                                   [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
  33                                   [IntrNoFree, IntrWillReturn]>;
  34
  35 def int_aarch64_clrex : Intrinsic<[]>;
  36
  37 def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  38                                 LLVMMatchType<0>], [IntrNoMem]>;
  39 def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  40                                 LLVMMatchType<0>], [IntrNoMem]>;
  41
  42 def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
  43
  44 def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
  45 def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
  46
  47 //===----------------------------------------------------------------------===//
  48 // HINT
  49
  50 def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;
  51
  52 //===----------------------------------------------------------------------===//
  53 // Data Barrier Instructions
  54
  55 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">,
  56                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
  57 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">,
  58                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
  59 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">,
  60                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
  61
  62 // A space-consuming intrinsic primarily for testing block and jump table
  63 // placements. The first argument is the number of bytes this "instruction"
  64 // takes up, the second and return value are essentially chains, used to force
  65 // ordering during ISel.
  66 def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
  67
  68 }
  69
  70 //===----------------------------------------------------------------------===//
  71 // Advanced SIMD (NEON)
  72
  73 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  74   class AdvSIMD_2Scalar_Float_Intrinsic
  75     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  76                 [IntrNoMem]>;
  77
  78   class AdvSIMD_FPToIntRounding_Intrinsic
  79     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
  80
  81   class AdvSIMD_1IntArg_Intrinsic
  82     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  83   class AdvSIMD_1FloatArg_Intrinsic
  84     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  85   class AdvSIMD_1VectorArg_Intrinsic
  86     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  87   class AdvSIMD_1VectorArg_Expand_Intrinsic
  88     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  89   class AdvSIMD_1VectorArg_Long_Intrinsic
  90     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
  91   class AdvSIMD_1IntArg_Narrow_Intrinsic
  92     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
  93   class AdvSIMD_1VectorArg_Narrow_Intrinsic
  94     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
  95   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
  96     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  97   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
  98     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  99
 100   class AdvSIMD_2IntArg_Intrinsic
 101     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
 102                 [IntrNoMem]>;
 103   class AdvSIMD_2FloatArg_Intrinsic
 104     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
 105                 [IntrNoMem]>;
 106   class AdvSIMD_2VectorArg_Intrinsic
 107     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
 108                 [IntrNoMem]>;
 109   class AdvSIMD_2VectorArg_Compare_Intrinsic
 110     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
 111                 [IntrNoMem]>;
 112   class AdvSIMD_2Arg_FloatCompare_Intrinsic
 113     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
 114                 [IntrNoMem]>;
 115   class AdvSIMD_2VectorArg_Long_Intrinsic
 116     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 117                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
 118                 [IntrNoMem]>;
 119   class AdvSIMD_2VectorArg_Wide_Intrinsic
 120     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 121                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
 122                 [IntrNoMem]>;
 123   class AdvSIMD_2VectorArg_Narrow_Intrinsic
 124     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 125                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
 126                 [IntrNoMem]>;
 127   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
 128     : DefaultAttrsIntrinsic<[llvm_anyint_ty],
 129                 [LLVMExtendedType<0>, llvm_i32_ty],
 130                 [IntrNoMem]>;
 131   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
 132     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 133                 [llvm_anyvector_ty],
 134                 [IntrNoMem]>;
 135   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
 136     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 137                 [LLVMTruncatedType<0>],
 138                 [IntrNoMem]>;
 139   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
 140     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 141                 [LLVMTruncatedType<0>, llvm_i32_ty],
 142                 [IntrNoMem]>;
 143   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
 144     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 145                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
 146                 [IntrNoMem]>;
 147   class AdvSIMD_2VectorArg_Lane_Intrinsic
 148     : DefaultAttrsIntrinsic<[llvm_anyint_ty],
 149                 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
 150                 [IntrNoMem]>;
 151
 152   class AdvSIMD_3VectorArg_Intrinsic
 153       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 154                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 155                [IntrNoMem]>;
 156   class AdvSIMD_3VectorArg_Scalar_Intrinsic
 157       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 158                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
 159                [IntrNoMem]>;
 160   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
 161       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 162                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
 163                 LLVMMatchType<1>], [IntrNoMem]>;
 164   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
 165     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 166                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
 167                 [IntrNoMem]>;
 168   class AdvSIMD_CvtFxToFP_Intrinsic
 169     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
 170                 [IntrNoMem]>;
 171   class AdvSIMD_CvtFPToFx_Intrinsic
 172     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
 173                 [IntrNoMem]>;
 174
 175   class AdvSIMD_1Arg_Intrinsic
 176     : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 177
 178   class AdvSIMD_Dot_Intrinsic
 179     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 180                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 181                 [IntrNoMem]>;
 182
 183   class AdvSIMD_FP16FML_Intrinsic
 184     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 185                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 186                 [IntrNoMem]>;
 187
 188   class AdvSIMD_MatMul_Intrinsic
 189     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 190                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 191                 [IntrNoMem]>;
 192
 193   class AdvSIMD_FML_Intrinsic
 194     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 195                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 196                 [IntrNoMem]>;
 197
 198   class AdvSIMD_BF16FML_Intrinsic
 199     : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
 200                 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
 201                 [IntrNoMem]>;
 202 }
 203
 204 // Arithmetic ops
 205
 206 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
 207   // Vector Add Across Lanes
 208   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 209   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 210   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 211
 212   // Vector Long Add Across Lanes
 213   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 214   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 215
 216   // Vector Halving Add
 217   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
 218   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
 219
 220   // Vector Rounding Halving Add
 221   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
 222   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
 223
 224   // Vector Saturating Add
 225   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
 226   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
 227   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
 228   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
 229
 230   // Vector Add High-Half
 231   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 232   // header is no longer supported.
 233   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 234
 235   // Vector Rounding Add High-Half
 236   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 237
 238   // Vector Saturating Doubling Multiply High
 239   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
 240   def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
 241   def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
 242
 243   // Vector Saturating Rounding Doubling Multiply High
 244   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
 245   def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
 246   def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
 247
 248   // Vector Polynominal Multiply
 249   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
 250
 251   // Vector Long Multiply
 252   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
 253   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
 254   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 255
 256   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
 257   // it with a v16i8.
 258   def int_aarch64_neon_pmull64 :
 259         DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 260
 261   // Vector Extending Multiply
 262   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
 263     let IntrProperties = [IntrNoMem, Commutative];
 264   }
 265
 266   // Vector Saturating Doubling Long Multiply
 267   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 268   def int_aarch64_neon_sqdmulls_scalar
 269     : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 270
 271   // Vector Halving Subtract
 272   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
 273   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
 274
 275   // Vector Saturating Subtract
 276   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
 277   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
 278
 279   // Vector Subtract High-Half
 280   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 281   // header is no longer supported.
 282   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 283
 284   // Vector Rounding Subtract High-Half
 285   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 286
 287   // Vector Compare Absolute Greater-than-or-equal
 288   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 289
 290   // Vector Compare Absolute Greater-than
 291   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 292
 293   // Vector Absolute Difference
 294   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
 295   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
 296   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
 297
 298   // Scalar Absolute Difference
 299   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
 300
 301   // Vector Max
 302   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
 303   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
 304   def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
 305   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
 306
 307   // Vector Max Across Lanes
 308   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 309   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 310   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 311   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 312
 313   // Vector Min
 314   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
 315   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
 316   def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
 317   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
 318
 319   // Vector Min/Max Number
 320   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
 321   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
 322
 323   // Vector Min Across Lanes
 324   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 325   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 326   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 327   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 328
 329   // Pairwise Add
 330   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
 331   def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
 332
 333   // Long Pairwise Add
 334   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
 335   // uaddlp, but tblgen's type inference currently can't handle the
 336   // pattern fragments this ends up generating.
 337   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 338   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 339
 340   // Folding Maximum
 341   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
 342   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
 343   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
 344
 345   // Folding Minimum
 346   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
 347   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
 348   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
 349
 350   // Reciprocal Estimate/Step
 351   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
 352   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
 353
 354   // Reciprocal Exponent
 355   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
 356
 357   // Vector Saturating Shift Left
 358   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
 359   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
 360
 361   // Vector Rounding Shift Left
 362   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
 363   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
 364
 365   // Vector Saturating Rounding Shift Left
 366   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
 367   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
 368
 369   // Vector Signed->Unsigned Shift Left by Constant
 370   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
 371
 372   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
 373   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 374
 375   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
 376   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 377
 378   // Vector Narrowing Shift Right by Constant
 379   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 380   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 381
 382   // Vector Rounding Narrowing Shift Right by Constant
 383   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 384
 385   // Vector Rounding Narrowing Saturating Shift Right by Constant
 386   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 387   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 388
 389   // Vector Shift Left
 390   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
 391   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
 392
 393   // Vector Widening Shift Left by Constant
 394   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
 395   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 396   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 397
 398   // Vector Shift Right by Constant and Insert
 399   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 400
 401   // Vector Shift Left by Constant and Insert
 402   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 403
 404   // Vector Saturating Narrow
 405   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
 406   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
 407   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 408   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 409
 410   // Vector Saturating Extract and Unsigned Narrow
 411   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
 412   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 413
 414   // Vector Absolute Value
 415   def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
 416
 417   // Vector Saturating Absolute Value
 418   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
 419
 420   // Vector Saturating Negation
 421   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
 422
 423   // Vector Count Leading Sign Bits
 424   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
 425
 426   // Vector Reciprocal Estimate
 427   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
 428   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
 429
 430   // Vector Square Root Estimate
 431   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
 432   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
 433
 434   // Vector Bitwise Reverse
 435   def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
 436
 437   // Vector Conversions Between Half-Precision and Single-Precision.
 438   def int_aarch64_neon_vcvtfp2hf
 439     : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 440   def int_aarch64_neon_vcvthf2fp
 441     : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
 442
 443   // Vector Conversions Between Floating-point and Fixed-point.
 444   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
 445   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
 446   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 447   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 448
 449   // Vector FP->Int Conversions
 450   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
 451   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
 452   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
 453   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
 454   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
 455   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
 456   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
 457   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
 458   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
 459   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 460
 461   // Vector FP Rounding: only ties to even is unrepresented by a normal
 462   // intrinsic.
 463   def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
 464
 465   // Scalar FP->Int conversions
 466
 467   // Vector FP Inexact Narrowing
 468   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
 469
 470   // Scalar FP Inexact Narrowing
 471   def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty],
 472                                         [IntrNoMem]>;
 473
 474   // v8.2-A Dot Product
 475   def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
 476   def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
 477
 478 // v8.6-A Matrix Multiply Intrinsics
 479   def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
 480   def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
 481   def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
 482   def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
 483   def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
 484   def int_aarch64_neon_bfmmla
 485     : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
 486                 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
 487                 [IntrNoMem]>;
 488   def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic;
 489   def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic;
 490
 491
 492   // v8.6-A Bfloat Intrinsics
 493   def int_aarch64_neon_bfcvt
 494     : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
 495   def int_aarch64_neon_bfcvtn
 496     : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 497   def int_aarch64_neon_bfcvtn2
 498     : DefaultAttrsIntrinsic<[llvm_v8bf16_ty],
 499                 [llvm_v8bf16_ty, llvm_v4f32_ty],
 500                 [IntrNoMem]>;
 501
 502   // v8.2-A FP16 Fused Multiply-Add Long
 503   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
 504   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
 505   def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
 506   def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
 507
 508   // v8.3-A Floating-point complex add
 509   def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
 510   def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
 511
 512   def int_aarch64_neon_vcmla_rot0   : AdvSIMD_3VectorArg_Intrinsic;
 513   def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;
 514   def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;
 515   def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;
 516 }
 517
 518 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 519   class AdvSIMD_2Vector2Index_Intrinsic
 520     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 521                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
 522                 [IntrNoMem]>;
 523 }
 524
 525 // Vector element to element moves
 526 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
 527
 528 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 529   class AdvSIMD_1Vec_Load_Intrinsic
 530       : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
 531                   [IntrReadMem, IntrArgMemOnly]>;
 532   class AdvSIMD_1Vec_Store_Lane_Intrinsic
 533     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
 534                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 535
 536   class AdvSIMD_2Vec_Load_Intrinsic
 537     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
 538                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 539                 [IntrReadMem, IntrArgMemOnly]>;
 540   class AdvSIMD_2Vec_Load_Lane_Intrinsic
 541     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
 542                 [LLVMMatchType<0>, llvm_anyvector_ty,
 543                  llvm_i64_ty, llvm_anyptr_ty],
 544                 [IntrReadMem, IntrArgMemOnly]>;
 545   class AdvSIMD_2Vec_Store_Intrinsic
 546     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 547                      LLVMAnyPointerType<LLVMMatchType<0>>],
 548                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 549   class AdvSIMD_2Vec_Store_Lane_Intrinsic
 550     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 551                  llvm_i64_ty, llvm_anyptr_ty],
 552                 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 553
 554   class AdvSIMD_3Vec_Load_Intrinsic
 555     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
 556                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 557                 [IntrReadMem, IntrArgMemOnly]>;
 558   class AdvSIMD_3Vec_Load_Lane_Intrinsic
 559     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 560                 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
 561                  llvm_i64_ty, llvm_anyptr_ty],
 562                 [IntrReadMem, IntrArgMemOnly]>;
 563   class AdvSIMD_3Vec_Store_Intrinsic
 564     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 565                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
 566                 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 567   class AdvSIMD_3Vec_Store_Lane_Intrinsic
 568     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,
 569                  LLVMMatchType<0>, LLVMMatchType<0>,
 570                  llvm_i64_ty, llvm_anyptr_ty],
 571                 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 572
 573   class AdvSIMD_4Vec_Load_Intrinsic
 574     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 575                  LLVMMatchType<0>, llvm_anyvector_ty],
 576                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 577                 [IntrReadMem, IntrArgMemOnly]>;
 578   class AdvSIMD_4Vec_Load_Lane_Intrinsic
 579     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 580                  LLVMMatchType<0>, LLVMMatchType<0>],
 581                 [LLVMMatchType<0>, LLVMMatchType<0>,
 582                  LLVMMatchType<0>, llvm_anyvector_ty,
 583                  llvm_i64_ty, llvm_anyptr_ty],
 584                 [IntrReadMem, IntrArgMemOnly]>;
 585   class AdvSIMD_4Vec_Store_Intrinsic
 586     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 587                  LLVMMatchType<0>, LLVMMatchType<0>,
 588                  LLVMAnyPointerType<LLVMMatchType<0>>],
 589                 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 590   class AdvSIMD_4Vec_Store_Lane_Intrinsic
 591     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 592                  LLVMMatchType<0>, LLVMMatchType<0>,
 593                  llvm_i64_ty, llvm_anyptr_ty],
 594                 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 595 }
 596
 597 // Memory ops
 598
 599 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
 600 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
 601 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
 602
 603 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
 604 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
 605 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
 606
 607 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
 608 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
 609 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
 610
 611 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
 612 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
 613 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
 614
 615 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
 616 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
 617 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
 618
 619 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
 620 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
 621 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
 622
 623 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
 624 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
 625 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
 626
 627 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 628   class AdvSIMD_Tbl1_Intrinsic
 629     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
 630                 [IntrNoMem]>;
 631   class AdvSIMD_Tbl2_Intrinsic
 632     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 633                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
 634   class AdvSIMD_Tbl3_Intrinsic
 635     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 636                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 637                  LLVMMatchType<0>],
 638                 [IntrNoMem]>;
 639   class AdvSIMD_Tbl4_Intrinsic
 640     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 641                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 642                  LLVMMatchType<0>],
 643                 [IntrNoMem]>;
 644
 645   class AdvSIMD_Tbx1_Intrinsic
 646     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 647                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
 648                 [IntrNoMem]>;
 649   class AdvSIMD_Tbx2_Intrinsic
 650     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 651                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 652                  LLVMMatchType<0>],
 653                 [IntrNoMem]>;
 654   class AdvSIMD_Tbx3_Intrinsic
 655     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 656                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 657                  llvm_v16i8_ty, LLVMMatchType<0>],
 658                 [IntrNoMem]>;
 659   class AdvSIMD_Tbx4_Intrinsic
 660     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 661                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 662                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
 663                 [IntrNoMem]>;
 664 }
 665 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
 666 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
 667 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
 668 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
 669
 670 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 671 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 672 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 673 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 674
 675 let TargetPrefix = "aarch64" in {
 676   class FPCR_Get_Intrinsic
 677     : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
 678 }
 679
 680 // FPCR
 681 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
 682
 683 let TargetPrefix = "aarch64" in {
 684   class Crypto_AES_DataKey_Intrinsic
 685     : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 686
 687   class Crypto_AES_Data_Intrinsic
 688     : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 689
 690   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 691   // (v4i32).
 692   class Crypto_SHA_5Hash4Schedule_Intrinsic
 693     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
 694                 [IntrNoMem]>;
 695
 696   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 697   // (v4i32).
 698   class Crypto_SHA_1Hash_Intrinsic
 699     : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 700
 701   // SHA intrinsic taking 8 words of the schedule
 702   class Crypto_SHA_8Schedule_Intrinsic
 703     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 704
 705   // SHA intrinsic taking 12 words of the schedule
 706   class Crypto_SHA_12Schedule_Intrinsic
 707     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 708                 [IntrNoMem]>;
 709
 710   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
 711   class Crypto_SHA_8Hash4Schedule_Intrinsic
 712     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 713                 [IntrNoMem]>;
 714 }
 715
 716 // AES
 717 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
 718 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
 719 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
 720 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
 721
 722 // SHA1
 723 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 724 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 725 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 726 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
 727
 728 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
 729 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
 730
 731 // SHA256
 732 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
 733 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
 734 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
 735 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
 736
 737 //===----------------------------------------------------------------------===//
 738 // CRC32
 739
 740 let TargetPrefix = "aarch64" in {
 741
 742 def int_aarch64_crc32b  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 743     [IntrNoMem]>;
 744 def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 745     [IntrNoMem]>;
 746 def int_aarch64_crc32h  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 747     [IntrNoMem]>;
 748 def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 749     [IntrNoMem]>;
 750 def int_aarch64_crc32w  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 751     [IntrNoMem]>;
 752 def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 753     [IntrNoMem]>;
 754 def int_aarch64_crc32x  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 755     [IntrNoMem]>;
 756 def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 757     [IntrNoMem]>;
 758 }
 759
 760 //===----------------------------------------------------------------------===//
 761 // Memory Tagging Extensions (MTE) Intrinsics
 762 let TargetPrefix = "aarch64" in {
 763 def int_aarch64_irg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 764     [IntrNoMem, IntrHasSideEffects]>;
 765 def int_aarch64_addg  : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 766     [IntrNoMem]>;
 767 def int_aarch64_gmi   : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
 768     [IntrNoMem]>;
 769 def int_aarch64_ldg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
 770     [IntrReadMem]>;
 771 def int_aarch64_stg   : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
 772     [IntrWriteMem]>;
 773 def int_aarch64_subp :  DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
 774     [IntrNoMem]>;
 775
 776 // The following are codegen-only intrinsics for stack instrumentation.
 777
 778 // Generate a randomly tagged stack base pointer.
 779 def int_aarch64_irg_sp   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty],
 780     [IntrNoMem, IntrHasSideEffects]>;
 781
 782 // Transfer pointer tag with offset.
 783 // ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
 784 // * address is the address in ptr0
 785 // * tag is a function of (tag in baseptr, tag_offset).
 786 // ** Beware, this is not the same function as implemented by the ADDG instruction!
 787 //    Backend optimizations may change tag_offset; the only guarantee is that calls
 788 //    to tagp with the same pair of (baseptr, tag_offset) will produce pointers
 789 //    with the same tag value, assuming the set of excluded tags has not changed.
 790 // Address bits in baseptr and tag bits in ptr0 are ignored.
 791 // When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
 792 //   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
 793 // It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
 794 def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
 795     [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 796
 797 // Update allocation tags for the memory range to match the tag in the pointer argument.
 798 def int_aarch64_settag  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 799     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 800
 801 // Update allocation tags for the memory range to match the tag in the pointer argument,
 802 // and set memory contents to zero.
 803 def int_aarch64_settag_zero  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 804     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 805
 806 // Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
 807 def int_aarch64_stgp  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
 808     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 809 }
 810
 811 // Transactional Memory Extension (TME) Intrinsics
 812 let TargetPrefix = "aarch64" in {
 813 def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
 814                          Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;
 815
 816 def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>;
 817
 818 def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
 819                           Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>;
 820
 821 def int_aarch64_ttest   : GCCBuiltin<"__builtin_arm_ttest">,
 822                           Intrinsic<[llvm_i64_ty], [],
 823                                     [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
 824
 825 // Armv8.7-A load/store 64-byte intrinsics
 826 defvar data512 = !listsplat(llvm_i64_ty, 8);
 827 def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>;
 828 def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;
 829 def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
 830 def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
 831
 832 }
 833
 834 def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
 835 def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
 836 def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
 837 def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
 838 def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
 839 def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
 840 def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
 841 def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
 842 def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
 843 def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
 844 def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
 845
 846 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 847
 848   class AdvSIMD_SVE_Create_2Vector_Tuple
 849     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 850                 [llvm_anyvector_ty, LLVMMatchType<1>],
 851                 [IntrReadMem]>;
 852
 853   class AdvSIMD_SVE_Create_3Vector_Tuple
 854     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 855                 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
 856                 [IntrReadMem]>;
 857
 858   class AdvSIMD_SVE_Create_4Vector_Tuple
 859     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 860                 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
 861                  LLVMMatchType<1>],
 862                 [IntrReadMem]>;
 863
 864   class AdvSIMD_SVE_Set_Vector_Tuple
 865     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 866                 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
 867                 [IntrReadMem, ImmArg<ArgIndex<1>>]>;
 868
 869   class AdvSIMD_SVE_Get_Vector_Tuple
 870     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
 871                 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 872
 873   class AdvSIMD_ManyVec_PredLoad_Intrinsic
 874     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
 875                 [IntrReadMem, IntrArgMemOnly]>;
 876
 877   class AdvSIMD_1Vec_PredLoad_Intrinsic
 878     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 879                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 880                  LLVMPointerToElt<0>],
 881                 [IntrReadMem, IntrArgMemOnly]>;
 882
 883   class AdvSIMD_1Vec_PredStore_Intrinsic
 884     : DefaultAttrsIntrinsic<[],
 885                 [llvm_anyvector_ty,
 886                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 887                  LLVMPointerToElt<0>],
 888                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 889
 890   class AdvSIMD_2Vec_PredStore_Intrinsic
 891       : DefaultAttrsIntrinsic<[],
 892                   [llvm_anyvector_ty, LLVMMatchType<0>,
 893                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
 894                   [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 895
 896   class AdvSIMD_3Vec_PredStore_Intrinsic
 897       : DefaultAttrsIntrinsic<[],
 898                   [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
 899                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
 900                   [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 901
 902   class AdvSIMD_4Vec_PredStore_Intrinsic
 903       : DefaultAttrsIntrinsic<[],
 904                   [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
 905                    LLVMMatchType<0>,
 906                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
 907                   [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 908
 909   class AdvSIMD_SVE_Index_Intrinsic
 910     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 911                 [LLVMVectorElementType<0>,
 912                  LLVMVectorElementType<0>],
 913                 [IntrNoMem]>;
 914
 915   class AdvSIMD_Merged1VectorArg_Intrinsic
 916     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 917                 [LLVMMatchType<0>,
 918                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 919                  LLVMMatchType<0>],
 920                 [IntrNoMem]>;
 921
 922   class AdvSIMD_2VectorArgIndexed_Intrinsic
 923     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 924                 [LLVMMatchType<0>,
 925                  LLVMMatchType<0>,
 926                  llvm_i32_ty],
 927                 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 928
 929   class AdvSIMD_3VectorArgIndexed_Intrinsic
 930     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 931                 [LLVMMatchType<0>,
 932                  LLVMMatchType<0>,
 933                  LLVMMatchType<0>,
 934                  llvm_i32_ty],
 935                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
 936
 937   class AdvSIMD_Pred1VectorArg_Intrinsic
 938     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 939                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 940                  LLVMMatchType<0>],
 941                 [IntrNoMem]>;
 942
 943   class AdvSIMD_Pred2VectorArg_Intrinsic
 944     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 945                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 946                  LLVMMatchType<0>,
 947                  LLVMMatchType<0>],
 948                 [IntrNoMem]>;
 949
 950   class AdvSIMD_Pred3VectorArg_Intrinsic
 951     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 952                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 953                  LLVMMatchType<0>,
 954                  LLVMMatchType<0>,
 955                  LLVMMatchType<0>],
 956                 [IntrNoMem]>;
 957
 958   class AdvSIMD_SVE_Compare_Intrinsic
 959     : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
 960                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 961                  llvm_anyvector_ty,
 962                  LLVMMatchType<0>],
 963                 [IntrNoMem]>;
 964
 965   class AdvSIMD_SVE_CompareWide_Intrinsic
 966     : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
 967                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 968                  llvm_anyvector_ty,
 969                  llvm_nxv2i64_ty],
 970                 [IntrNoMem]>;
 971
 972   class AdvSIMD_SVE_Saturating_Intrinsic
 973     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 974                 [LLVMMatchType<0>,
 975                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
 976                 [IntrNoMem]>;
 977
 978   class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
 979     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 980                 [LLVMMatchType<0>,
 981                  llvm_i32_ty,
 982                  llvm_i32_ty],
 983                 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
 984
 985   class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
 986     : DefaultAttrsIntrinsic<[T],
 987                 [T, llvm_anyvector_ty],
 988                 [IntrNoMem]>;
 989
 990   class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
 991     : DefaultAttrsIntrinsic<[T],
 992                 [T, llvm_i32_ty, llvm_i32_ty],
 993                 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
 994
 995   class AdvSIMD_SVE_CNT_Intrinsic
 996     : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>],
 997                 [LLVMVectorOfBitcastsToInt<0>,
 998                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 999                  llvm_anyvector_ty],
1000                 [IntrNoMem]>;
1001
1002   class AdvSIMD_SVE_ReduceWithInit_Intrinsic
1003     : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
1004                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1005                  LLVMVectorElementType<0>,
1006                  llvm_anyvector_ty],
1007                 [IntrNoMem]>;
1008
1009   class AdvSIMD_SVE_ShiftByImm_Intrinsic
1010     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1011                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1012                  LLVMMatchType<0>,
1013                  llvm_i32_ty],
1014                 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1015
1016   class AdvSIMD_SVE_ShiftWide_Intrinsic
1017     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1018                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1019                  LLVMMatchType<0>,
1020                  llvm_nxv2i64_ty],
1021                 [IntrNoMem]>;
1022
1023   class AdvSIMD_SVE_Unpack_Intrinsic
1024     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1025                [LLVMSubdivide2VectorType<0>],
1026                [IntrNoMem]>;
1027
1028   class AdvSIMD_SVE_CADD_Intrinsic
1029     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1030                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1031                  LLVMMatchType<0>,
1032                  LLVMMatchType<0>,
1033                  llvm_i32_ty],
1034                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1035
1036   class AdvSIMD_SVE_CMLA_Intrinsic
1037     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1038                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1039                  LLVMMatchType<0>,
1040                  LLVMMatchType<0>,
1041                  LLVMMatchType<0>,
1042                  llvm_i32_ty],
1043                 [IntrNoMem, ImmArg<ArgIndex<4>>]>;
1044
1045   class AdvSIMD_SVE_CMLA_LANE_Intrinsic
1046     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1047                 [LLVMMatchType<0>,
1048                  LLVMMatchType<0>,
1049                  LLVMMatchType<0>,
1050                  llvm_i32_ty,
1051                  llvm_i32_ty],
1052                 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1053
1054   class AdvSIMD_SVE_DUP_Intrinsic
1055     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1056                 [LLVMMatchType<0>,
1057                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1058                  LLVMVectorElementType<0>],
1059                 [IntrNoMem]>;
1060
1061   class AdvSIMD_SVE_DUP_Unpred_Intrinsic
1062     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
1063                 [IntrNoMem]>;
1064
1065   class AdvSIMD_SVE_DUPQ_Intrinsic
1066     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1067                 [LLVMMatchType<0>,
1068                  llvm_i64_ty],
1069                 [IntrNoMem]>;
1070
1071   class AdvSIMD_SVE_EXPA_Intrinsic
1072     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1073                 [LLVMVectorOfBitcastsToInt<0>],
1074                 [IntrNoMem]>;
1075
1076   class AdvSIMD_SVE_FCVT_Intrinsic
1077     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1078                 [LLVMMatchType<0>,
1079                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1080                  llvm_anyvector_ty],
1081                 [IntrNoMem]>;
1082
1083   class AdvSIMD_SVE_FCVTZS_Intrinsic
1084     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1085                 [LLVMVectorOfBitcastsToInt<0>,
1086                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1087                  llvm_anyvector_ty],
1088                 [IntrNoMem]>;
1089
1090   class AdvSIMD_SVE_INSR_Intrinsic
1091     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1092                 [LLVMMatchType<0>,
1093                  LLVMVectorElementType<0>],
1094                 [IntrNoMem]>;
1095
1096   class AdvSIMD_SVE_PTRUE_Intrinsic
1097     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1098                 [llvm_i32_ty],
1099                 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1100
1101   class AdvSIMD_SVE_PUNPKHI_Intrinsic
1102     : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>],
1103                 [llvm_anyvector_ty],
1104                 [IntrNoMem]>;
1105
1106   class AdvSIMD_SVE_SCALE_Intrinsic
1107     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1108                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1109                  LLVMMatchType<0>,
1110                  LLVMVectorOfBitcastsToInt<0>],
1111                 [IntrNoMem]>;
1112
1113   class AdvSIMD_SVE_SCVTF_Intrinsic
1114     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1115                 [LLVMMatchType<0>,
1116                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1117                  llvm_anyvector_ty],
1118                 [IntrNoMem]>;
1119
1120   class AdvSIMD_SVE_TSMUL_Intrinsic
1121     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1122                 [LLVMMatchType<0>,
1123                  LLVMVectorOfBitcastsToInt<0>],
1124                 [IntrNoMem]>;
1125
1126   class AdvSIMD_SVE_CNTB_Intrinsic
1127     : DefaultAttrsIntrinsic<[llvm_i64_ty],
1128                 [llvm_i32_ty],
1129                 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1130
1131   class AdvSIMD_SVE_CNTP_Intrinsic
1132     : DefaultAttrsIntrinsic<[llvm_i64_ty],
1133                 [llvm_anyvector_ty, LLVMMatchType<0>],
1134                 [IntrNoMem]>;
1135
1136   class AdvSIMD_SVE_DOT_Intrinsic
1137     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1138                 [LLVMMatchType<0>,
1139                  LLVMSubdivide4VectorType<0>,
1140                  LLVMSubdivide4VectorType<0>],
1141                 [IntrNoMem]>;
1142
1143   class AdvSIMD_SVE_DOT_Indexed_Intrinsic
1144     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1145                 [LLVMMatchType<0>,
1146                  LLVMSubdivide4VectorType<0>,
1147                  LLVMSubdivide4VectorType<0>,
1148                  llvm_i32_ty],
1149                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1150
1151   class AdvSIMD_SVE_PTEST_Intrinsic
1152     : DefaultAttrsIntrinsic<[llvm_i1_ty],
1153                 [llvm_anyvector_ty,
1154                  LLVMMatchType<0>],
1155                 [IntrNoMem]>;
1156
1157   class AdvSIMD_SVE_TBL_Intrinsic
1158     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1159                 [LLVMMatchType<0>,
1160                  LLVMVectorOfBitcastsToInt<0>],
1161                 [IntrNoMem]>;
1162
1163   class AdvSIMD_SVE2_TBX_Intrinsic
1164     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1165                 [LLVMMatchType<0>,
1166                  LLVMMatchType<0>,
1167                  LLVMVectorOfBitcastsToInt<0>],
1168                 [IntrNoMem]>;
1169
1170   class SVE2_1VectorArg_Long_Intrinsic
1171     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1172                 [LLVMSubdivide2VectorType<0>,
1173                  llvm_i32_ty],
1174                 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1175
1176   class SVE2_2VectorArg_Long_Intrinsic
1177     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1178                 [LLVMSubdivide2VectorType<0>,
1179                  LLVMSubdivide2VectorType<0>],
1180                 [IntrNoMem]>;
1181
1182   class SVE2_2VectorArgIndexed_Long_Intrinsic
1183   : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1184               [LLVMSubdivide2VectorType<0>,
1185                LLVMSubdivide2VectorType<0>,
1186                llvm_i32_ty],
1187               [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1188
1189   class SVE2_2VectorArg_Wide_Intrinsic
1190     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1191                 [LLVMMatchType<0>,
1192                  LLVMSubdivide2VectorType<0>],
1193                 [IntrNoMem]>;
1194
1195   class SVE2_2VectorArg_Pred_Long_Intrinsic
1196     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1197                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1198                  LLVMMatchType<0>,
1199                  LLVMSubdivide2VectorType<0>],
1200                 [IntrNoMem]>;
1201
1202   class SVE2_3VectorArg_Long_Intrinsic
1203     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1204                 [LLVMMatchType<0>,
1205                  LLVMSubdivide2VectorType<0>,
1206                  LLVMSubdivide2VectorType<0>],
1207                 [IntrNoMem]>;
1208
1209   class SVE2_3VectorArgIndexed_Long_Intrinsic
1210     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1211                 [LLVMMatchType<0>,
1212                  LLVMSubdivide2VectorType<0>,
1213                  LLVMSubdivide2VectorType<0>,
1214                  llvm_i32_ty],
1215                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1216
1217   class SVE2_1VectorArg_Narrowing_Intrinsic
1218     : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1219                 [llvm_anyvector_ty],
1220                 [IntrNoMem]>;
1221
1222   class SVE2_Merged1VectorArg_Narrowing_Intrinsic
1223     : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1224                 [LLVMSubdivide2VectorType<0>,
1225                  llvm_anyvector_ty],
1226                 [IntrNoMem]>;
1227   class SVE2_2VectorArg_Narrowing_Intrinsic
1228       : DefaultAttrsIntrinsic<
1229             [LLVMSubdivide2VectorType<0>],
1230             [llvm_anyvector_ty, LLVMMatchType<0>],
1231             [IntrNoMem]>;
1232
1233   class SVE2_Merged2VectorArg_Narrowing_Intrinsic
1234       : DefaultAttrsIntrinsic<
1235             [LLVMSubdivide2VectorType<0>],
1236             [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
1237             [IntrNoMem]>;
1238
1239   class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
1240       : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1241                   [llvm_anyvector_ty, llvm_i32_ty],
1242                   [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1243
1244   class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
1245       : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1246                   [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
1247                    llvm_i32_ty],
1248                   [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1249
1250   class SVE2_CONFLICT_DETECT_Intrinsic
1251     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1252                 [LLVMAnyPointerType<llvm_any_ty>,
1253                  LLVMMatchType<1>]>;
1254
1255   class SVE2_3VectorArg_Indexed_Intrinsic
1256     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1257                 [LLVMMatchType<0>,
1258                  LLVMSubdivide2VectorType<0>,
1259                  LLVMSubdivide2VectorType<0>,
1260                  llvm_i32_ty],
1261                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1262
1263   class AdvSIMD_SVE_CDOT_LANE_Intrinsic
1264     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1265                 [LLVMMatchType<0>,
1266                  LLVMSubdivide4VectorType<0>,
1267                  LLVMSubdivide4VectorType<0>,
1268                  llvm_i32_ty,
1269                  llvm_i32_ty],
1270                 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1271
1272   // NOTE: There is no relationship between these intrinsics beyond an attempt
1273   // to reuse currently identical class definitions.
1274   class AdvSIMD_SVE_LOGB_Intrinsic  : AdvSIMD_SVE_CNT_Intrinsic;
1275   class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
1276   class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;
1277
1278   // This class of intrinsics are not intended to be useful within LLVM IR but
1279   // are instead here to support some of the more regid parts of the ACLE.
1280   class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN>
1281       : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
1282 }
1283
1284 //===----------------------------------------------------------------------===//
1285 // SVE
1286
1287 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
1288
1289 class AdvSIMD_SVE_Reduce_Intrinsic
1290   : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
1291               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1292                llvm_anyvector_ty],
1293               [IntrNoMem]>;
1294
1295 class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
1296   : DefaultAttrsIntrinsic<[llvm_i64_ty],
1297               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1298                llvm_anyvector_ty],
1299               [IntrNoMem]>;
1300
1301 class AdvSIMD_SVE_WHILE_Intrinsic
1302     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1303                 [llvm_anyint_ty, LLVMMatchType<1>],
1304                 [IntrNoMem]>;
1305
1306 class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
1307     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1308                 [
1309                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1310                   LLVMPointerToElt<0>,
1311                   LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1312                 ],
1313                 [IntrReadMem, IntrArgMemOnly]>;
1314
1315 class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
1316     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1317                 [
1318                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1319                   LLVMPointerToElt<0>,
1320                   LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1321                 ],
1322                 [IntrReadMem, IntrArgMemOnly]>;
1323
1324 class AdvSIMD_GatherLoad_VS_Intrinsic
1325     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1326                 [
1327                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1328                   llvm_anyvector_ty,
1329                   llvm_i64_ty
1330                 ],
1331                 [IntrReadMem]>;
1332
1333 class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
1334     : DefaultAttrsIntrinsic<[],
1335                [
1336                  llvm_anyvector_ty,
1337                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1338                  LLVMPointerToElt<0>,
1339                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1340                ],
1341                [IntrWriteMem, IntrArgMemOnly]>;
1342
1343 class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
1344     : DefaultAttrsIntrinsic<[],
1345                [
1346                  llvm_anyvector_ty,
1347                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1348                  LLVMPointerToElt<0>,
1349                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1350                ],
1351                [IntrWriteMem, IntrArgMemOnly]>;
1352
1353 class AdvSIMD_ScatterStore_VS_Intrinsic
1354     : DefaultAttrsIntrinsic<[],
1355                [
1356                  llvm_anyvector_ty,
1357                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1358                  llvm_anyvector_ty, llvm_i64_ty
1359                ],
1360                [IntrWriteMem]>;
1361
1362
1363 class SVE_gather_prf_SV
1364     : DefaultAttrsIntrinsic<[],
1365                 [
1366                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1367                   llvm_ptr_ty, // Base address
1368                   llvm_anyvector_ty, // Offsets
1369                   llvm_i32_ty // Prfop
1370                 ],
1371                 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;
1372
1373 class SVE_gather_prf_VS
1374     : DefaultAttrsIntrinsic<[],
1375                 [
1376                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1377                   llvm_anyvector_ty, // Base addresses
1378                   llvm_i64_ty, // Scalar offset
1379                   llvm_i32_ty // Prfop
1380                 ],
1381                 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;
1382
1383 class SVE_MatMul_Intrinsic
1384     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1385                 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
1386                 [IntrNoMem]>;
1387
1388 class SVE_4Vec_BF16
1389     : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
1390                 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
1391                 [IntrNoMem]>;
1392
1393 class SVE_4Vec_BF16_Indexed
1394     : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
1395                 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
1396                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1397
1398 //
1399 // Vector tuple creation intrinsics (ACLE)
1400 //
1401
1402 def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
1403 def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
1404 def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;
1405
1406 //
1407 // Vector tuple insertion/extraction intrinsics (ACLE)
1408 //
1409
1410 def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
1411 def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;
1412
1413 //
1414 // Loads
1415 //
1416
1417 def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;
1418
1419 def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1420 def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1421 def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1422
1423 def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1424 def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1425 def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1426
1427 def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1428 def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
1429
1430 //
1431 // Stores
1432 //
1433
1434 def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
1435 def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
1436 def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
1437 def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;
1438
1439 def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1440
1441 //
1442 // Prefetches
1443 //
1444
1445 def int_aarch64_sve_prf
1446   : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
1447                   [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
1448
1449 // Scalar + 32-bit scaled offset vector, zero extend, packed and
1450 // unpacked.
1451 def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
1452 def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
1453 def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
1454 def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;
1455
1456 // Scalar + 32-bit scaled offset vector, sign extend, packed and
1457 // unpacked.
1458 def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
1459 def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
1460 def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
1461 def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;
1462
1463 // Scalar + 64-bit scaled offset vector.
1464 def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
1465 def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
1466 def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
1467 def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;
1468
1469 // Vector + scalar.
1470 def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
1471 def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
1472 def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
1473 def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;
1474
1475 //
1476 // Scalar to vector operations
1477 //
1478
1479 def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
1480 def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;
1481
1482
1483 def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;
1484
1485 //
1486 // Address calculation
1487 //
1488
1489 def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
1490 def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
1491 def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
1492 def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;
1493
1494 //
1495 // Integer arithmetic
1496 //
1497
1498 def int_aarch64_sve_add   : AdvSIMD_Pred2VectorArg_Intrinsic;
1499 def int_aarch64_sve_sub   : AdvSIMD_Pred2VectorArg_Intrinsic;
1500 def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1501
1502 def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;
1503
1504 def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
1505 def int_aarch64_sve_mul_lane   : AdvSIMD_2VectorArgIndexed_Intrinsic;
1506 def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1507 def int_aarch64_sve_umulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1508
1509 def int_aarch64_sve_sdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1510 def int_aarch64_sve_udiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1511 def int_aarch64_sve_sdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1512 def int_aarch64_sve_udivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1513
1514 def int_aarch64_sve_smax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1515 def int_aarch64_sve_umax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1516 def int_aarch64_sve_smin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1517 def int_aarch64_sve_umin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1518 def int_aarch64_sve_sabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1519 def int_aarch64_sve_uabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1520
1521 def int_aarch64_sve_mad        : AdvSIMD_Pred3VectorArg_Intrinsic;
1522 def int_aarch64_sve_msb        : AdvSIMD_Pred3VectorArg_Intrinsic;
1523 def int_aarch64_sve_mla        : AdvSIMD_Pred3VectorArg_Intrinsic;
1524 def int_aarch64_sve_mla_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1525 def int_aarch64_sve_mls        : AdvSIMD_Pred3VectorArg_Intrinsic;
1526 def int_aarch64_sve_mls_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1527
1528 def int_aarch64_sve_saddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1529 def int_aarch64_sve_uaddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1530
1531 def int_aarch64_sve_smaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1532 def int_aarch64_sve_umaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1533 def int_aarch64_sve_sminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1534 def int_aarch64_sve_uminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1535
1536 def int_aarch64_sve_orv        : AdvSIMD_SVE_Reduce_Intrinsic;
1537 def int_aarch64_sve_eorv       : AdvSIMD_SVE_Reduce_Intrinsic;
1538 def int_aarch64_sve_andv       : AdvSIMD_SVE_Reduce_Intrinsic;
1539
1540 def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
1541 def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
1542
1543 def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
1544 def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1545
1546 def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
1547 def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1548
1549 def int_aarch64_sve_sqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1550 def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1551 def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1552 def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1553
1554 // Shifts
1555
1556 def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1557 def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1558 def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
1559 def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
1560 def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
1561 def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1562 def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1563 def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1564
1565 //
1566 // Integer comparisons
1567 //
1568
1569 def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1570 def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
1571 def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1572 def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
1573 def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
1574 def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
1575
1576 def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1577 def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1578 def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1579 def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1580 def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1581 def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1582 def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1583 def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1584 def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1585 def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1586
1587 //
1588 // Counting bits
1589 //
1590
1591 def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
1592 def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
1593 def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
1594
1595 //
1596 // Counting elements
1597 //
1598
1599 def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
1600 def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
1601 def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
1602 def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
1603
1604 def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
1605
1606 //
1607 // FFR manipulation
1608 //
1609
1610 def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], []>;
1611 def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
1612 def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  DefaultAttrsIntrinsic<[], []>;
1613 def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty]>;
1614
1615 //
1616 // Saturating scalar arithmetic
1617 //
1618
1619 def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1620 def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1621 def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1622 def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1623
1624 def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1625 def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1626 def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1627 def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1628 def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1629 def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1630 def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1631 def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1632 def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1633 def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1634
1635 def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1636 def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1637 def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1638 def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1639
1640 def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1641 def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1642 def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1643 def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1644 def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1645 def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1646 def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1647 def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1648 def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1649 def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1650
1651 def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1652 def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1653 def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1654 def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1655
1656 def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1657 def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1658 def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1659 def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1660 def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1661 def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1662 def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1663 def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1664 def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1665 def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1666
1667 def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1668 def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1669 def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1670 def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1671
1672 def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1673 def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1674 def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1675 def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1676 def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1677 def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1678 def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1679 def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1680 def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1681 def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1682
1683 //
1684 // Reversal
1685 //
1686
1687 def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
1688 def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
1689 def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
1690 def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;
1691
1692 //
1693 // Permutations and selection
1694 //
1695
1696 def int_aarch64_sve_clasta    : AdvSIMD_Pred2VectorArg_Intrinsic;
1697 def int_aarch64_sve_clasta_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1698 def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
1699 def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1700 def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
1701 def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
1702 def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
1703 def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
1704 def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
1705 def int_aarch64_sve_lastb     : AdvSIMD_SVE_Reduce_Intrinsic;
1706 def int_aarch64_sve_rev       : AdvSIMD_1VectorArg_Intrinsic;
1707 def int_aarch64_sve_splice    : AdvSIMD_Pred2VectorArg_Intrinsic;
1708 def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1709 def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1710 def int_aarch64_sve_tbl       : AdvSIMD_SVE_TBL_Intrinsic;
1711 def int_aarch64_sve_trn1      : AdvSIMD_2VectorArg_Intrinsic;
1712 def int_aarch64_sve_trn2      : AdvSIMD_2VectorArg_Intrinsic;
1713 def int_aarch64_sve_trn1q     : AdvSIMD_2VectorArg_Intrinsic;
1714 def int_aarch64_sve_trn2q     : AdvSIMD_2VectorArg_Intrinsic;
1715 def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1716 def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1717 def int_aarch64_sve_uzp1      : AdvSIMD_2VectorArg_Intrinsic;
1718 def int_aarch64_sve_uzp2      : AdvSIMD_2VectorArg_Intrinsic;
1719 def int_aarch64_sve_uzp1q     : AdvSIMD_2VectorArg_Intrinsic;
1720 def int_aarch64_sve_uzp2q     : AdvSIMD_2VectorArg_Intrinsic;
1721 def int_aarch64_sve_zip1      : AdvSIMD_2VectorArg_Intrinsic;
1722 def int_aarch64_sve_zip2      : AdvSIMD_2VectorArg_Intrinsic;
1723 def int_aarch64_sve_zip1q     : AdvSIMD_2VectorArg_Intrinsic;
1724 def int_aarch64_sve_zip2q     : AdvSIMD_2VectorArg_Intrinsic;
1725
1726 //
1727 // Logical operations
1728 //
1729
1730 def int_aarch64_sve_and  : AdvSIMD_Pred2VectorArg_Intrinsic;
1731 def int_aarch64_sve_bic  : AdvSIMD_Pred2VectorArg_Intrinsic;
1732 def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
1733 def int_aarch64_sve_eor  : AdvSIMD_Pred2VectorArg_Intrinsic;
1734 def int_aarch64_sve_not  : AdvSIMD_Merged1VectorArg_Intrinsic;
1735 def int_aarch64_sve_orr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1736
1737 //
1738 // Conversion
1739 //
1740
1741 def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1742 def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1743 def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1744 def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1745 def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1746 def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1747
1748 //
1749 // While comparisons
1750 //
1751
1752 def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
1753 def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
1754 def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
1755 def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
1756 def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
1757 def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
1758 def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
1759 def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;
1760
1761 //
1762 // Floating-point arithmetic
1763 //
1764
1765 def int_aarch64_sve_fabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1766 def int_aarch64_sve_fabs       : AdvSIMD_Merged1VectorArg_Intrinsic;
1767 def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1768 def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
1769 def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
1770 def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
1771 def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1772 def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1773 def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
1774 def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
1775 def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1776 def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1777 def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1778 def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1779 def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
1780 def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1781 def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
1782 def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1783 def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;
1784 def int_aarch64_sve_fmul       : AdvSIMD_Pred2VectorArg_Intrinsic;
1785 def int_aarch64_sve_fmulx      : AdvSIMD_Pred2VectorArg_Intrinsic;
1786 def int_aarch64_sve_fneg       : AdvSIMD_Merged1VectorArg_Intrinsic;
1787 def int_aarch64_sve_fmul_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
1788 def int_aarch64_sve_fnmad      : AdvSIMD_Pred3VectorArg_Intrinsic;
1789 def int_aarch64_sve_fnmla      : AdvSIMD_Pred3VectorArg_Intrinsic;
1790 def int_aarch64_sve_fnmls      : AdvSIMD_Pred3VectorArg_Intrinsic;
1791 def int_aarch64_sve_fnmsb      : AdvSIMD_Pred3VectorArg_Intrinsic;
1792 def int_aarch64_sve_frecpe_x   : AdvSIMD_1VectorArg_Intrinsic;
1793 def int_aarch64_sve_frecps_x   : AdvSIMD_2VectorArg_Intrinsic;
1794 def int_aarch64_sve_frecpx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1795 def int_aarch64_sve_frinta     : AdvSIMD_Merged1VectorArg_Intrinsic;
1796 def int_aarch64_sve_frinti     : AdvSIMD_Merged1VectorArg_Intrinsic;
1797 def int_aarch64_sve_frintm     : AdvSIMD_Merged1VectorArg_Intrinsic;
1798 def int_aarch64_sve_frintn     : AdvSIMD_Merged1VectorArg_Intrinsic;
1799 def int_aarch64_sve_frintp     : AdvSIMD_Merged1VectorArg_Intrinsic;
1800 def int_aarch64_sve_frintx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1801 def int_aarch64_sve_frintz     : AdvSIMD_Merged1VectorArg_Intrinsic;
1802 def int_aarch64_sve_frsqrte_x  : AdvSIMD_1VectorArg_Intrinsic;
1803 def int_aarch64_sve_frsqrts_x  : AdvSIMD_2VectorArg_Intrinsic;
1804 def int_aarch64_sve_fscale     : AdvSIMD_SVE_SCALE_Intrinsic;
1805 def int_aarch64_sve_fsqrt      : AdvSIMD_Merged1VectorArg_Intrinsic;
1806 def int_aarch64_sve_fsub       : AdvSIMD_Pred2VectorArg_Intrinsic;
1807 def int_aarch64_sve_fsubr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1808 def int_aarch64_sve_ftmad_x    : AdvSIMD_2VectorArgIndexed_Intrinsic;
1809 def int_aarch64_sve_ftsmul_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1810 def int_aarch64_sve_ftssel_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1811
1812 //
1813 // Floating-point reductions
1814 //
1815
1816 def int_aarch64_sve_fadda   : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1817 def int_aarch64_sve_faddv   : AdvSIMD_SVE_Reduce_Intrinsic;
1818 def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
1819 def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1820 def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
1821 def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1822
1823 //
1824 // Floating-point conversions
1825 //
1826
1827 def int_aarch64_sve_fcvt   : AdvSIMD_SVE_FCVT_Intrinsic;
1828 def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
1829 def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
1830 def int_aarch64_sve_scvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1831 def int_aarch64_sve_ucvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1832
1833 //
1834 // Floating-point comparisons
1835 //
1836
1837 def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
1838 def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;
1839
1840 def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1841 def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
1842 def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1843 def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
1844 def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;
1845
1846 def int_aarch64_sve_fcvtzs_i32f16   : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1847 def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1848 def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1849 def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1850
1851 def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<"svcvt_bf16_f32_m",   llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1852 def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1853
1854 def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1855 def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1856 def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1857 def int_aarch64_sve_fcvtzu_i64f32   : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1858
1859 def int_aarch64_sve_fcvt_f16f32     : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1860 def int_aarch64_sve_fcvt_f16f64     : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1861 def int_aarch64_sve_fcvt_f32f64     : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1862
1863 def int_aarch64_sve_fcvt_f32f16     : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1864 def int_aarch64_sve_fcvt_f64f16     : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1865 def int_aarch64_sve_fcvt_f64f32     : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1866
1867 def int_aarch64_sve_fcvtlt_f32f16   : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1868 def int_aarch64_sve_fcvtlt_f64f32   : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1869 def int_aarch64_sve_fcvtnt_f16f32   : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1870 def int_aarch64_sve_fcvtnt_f32f64   : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1871
1872 def int_aarch64_sve_fcvtx_f32f64    : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1873 def int_aarch64_sve_fcvtxnt_f32f64  : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1874
1875 def int_aarch64_sve_scvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1876 def int_aarch64_sve_scvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1877 def int_aarch64_sve_scvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1878 def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1879
1880 def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1881 def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1882 def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1883 def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1884
1885 //
1886 // Predicate creation
1887 //
1888
1889 def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
1890
1891 //
1892 // Predicate operations
1893 //
1894
1895 def int_aarch64_sve_and_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1896 def int_aarch64_sve_bic_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1897 def int_aarch64_sve_brka    : AdvSIMD_Merged1VectorArg_Intrinsic;
1898 def int_aarch64_sve_brka_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1899 def int_aarch64_sve_brkb    : AdvSIMD_Merged1VectorArg_Intrinsic;
1900 def int_aarch64_sve_brkb_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1901 def int_aarch64_sve_brkn_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1902 def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1903 def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1904 def int_aarch64_sve_eor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1905 def int_aarch64_sve_nand_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1906 def int_aarch64_sve_nor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1907 def int_aarch64_sve_orn_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1908 def int_aarch64_sve_orr_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1909 def int_aarch64_sve_pfirst  : AdvSIMD_Pred1VectorArg_Intrinsic;
1910 def int_aarch64_sve_pnext   : AdvSIMD_Pred1VectorArg_Intrinsic;
1911 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1912 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1913
1914 //
1915 // Testing predicates
1916 //
1917
1918 def int_aarch64_sve_ptest_any   : AdvSIMD_SVE_PTEST_Intrinsic;
1919 def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
1920 def int_aarch64_sve_ptest_last  : AdvSIMD_SVE_PTEST_Intrinsic;
1921
1922 //
1923 // Reinterpreting data
1924 //
1925
1926 def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1927                                                     [llvm_nxv16i1_ty],
1928                                                     [IntrNoMem]>;
1929
1930 def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty],
1931                                                   [llvm_anyvector_ty],
1932                                                   [IntrNoMem]>;
1933
1934 //
1935 // Gather loads: scalar base + vector offsets
1936 //
1937
1938 // 64 bit unscaled offsets
1939 def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1940
1941 // 64 bit scaled offsets
1942 def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1943
1944 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1945 def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1946 def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1947
1948 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1949 def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1950 def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1951
1952 //
1953 // Gather loads: vector base + scalar offset
1954 //
1955
1956 def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1957
1958
1959 //
1960 // First-faulting gather loads: scalar base + vector offsets
1961 //
1962
1963 // 64 bit unscaled offsets
1964 def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1965
1966 // 64 bit scaled offsets
1967 def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1968
1969 // 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1970 def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1971 def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1972
1973 // 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1974 def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1975 def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1976
1977 //
1978 // First-faulting gather loads: vector base + scalar offset
1979 //
1980
1981 def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1982
1983
1984 //
1985 // Non-temporal gather loads: scalar base + vector offsets
1986 //
1987
1988 // 64 bit unscaled offsets
1989 def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1990
1991 // 64 bit indices
1992 def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1993
1994 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
1995 def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1996
1997 //
1998 // Non-temporal gather loads: vector base + scalar offset
1999 //
2000
2001 def int_aarch64_sve_ldnt1_gather_scalar_offset  : AdvSIMD_GatherLoad_VS_Intrinsic;
2002
2003 //
2004 // Scatter stores: scalar base + vector offsets
2005 //
2006
2007 // 64 bit unscaled offsets
2008 def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2009
2010 // 64 bit scaled offsets
2011 def int_aarch64_sve_st1_scatter_index
2012     : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2013
2014 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2015 def int_aarch64_sve_st1_scatter_sxtw
2016     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2017
2018 def int_aarch64_sve_st1_scatter_uxtw
2019     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2020
2021 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2022 def int_aarch64_sve_st1_scatter_sxtw_index
2023     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2024
2025 def int_aarch64_sve_st1_scatter_uxtw_index
2026     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2027
2028 //
2029 // Scatter stores: vector base + scalar offset
2030 //
2031
2032 def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
2033
2034 //
2035 // Non-temporal scatter stores: scalar base + vector offsets
2036 //
2037
2038 // 64 bit unscaled offsets
2039 def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2040
2041 // 64 bit indices
2042 def int_aarch64_sve_stnt1_scatter_index
2043     : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2044
2045 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2046 def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2047
2048 //
2049 // Non-temporal scatter stores: vector base + scalar offset
2050 //
2051
2052 def int_aarch64_sve_stnt1_scatter_scalar_offset  : AdvSIMD_ScatterStore_VS_Intrinsic;
2053
2054 //
2055 // SVE2 - Uniform DSP operations
2056 //
2057
2058 def int_aarch64_sve_saba          : AdvSIMD_3VectorArg_Intrinsic;
2059 def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2060 def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2061 def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2062 def int_aarch64_sve_sli           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2063 def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
2064 def int_aarch64_sve_sqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2065 def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
2066 def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
2067 def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
2068 def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
2069 def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2070 def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
2071 def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2072 def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
2073 def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2074 def int_aarch64_sve_sqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2075 def int_aarch64_sve_sqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2076 def int_aarch64_sve_sqshlu        : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2077 def int_aarch64_sve_sqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2078 def int_aarch64_sve_sqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2079 def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2080 def int_aarch64_sve_sri           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2081 def int_aarch64_sve_srshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2082 def int_aarch64_sve_srshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2083 def int_aarch64_sve_srsra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2084 def int_aarch64_sve_ssra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2085 def int_aarch64_sve_suqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2086 def int_aarch64_sve_uaba          : AdvSIMD_3VectorArg_Intrinsic;
2087 def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2088 def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2089 def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2090 def int_aarch64_sve_uqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2091 def int_aarch64_sve_uqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2092 def int_aarch64_sve_uqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2093 def int_aarch64_sve_uqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2094 def int_aarch64_sve_uqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2095 def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
2096 def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2097 def int_aarch64_sve_urshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2098 def int_aarch64_sve_urshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2099 def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;
2100 def int_aarch64_sve_ursra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2101 def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2102 def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2103
2104 //
2105 // SVE2 - Widening DSP operations
2106 //
2107
2108 def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
2109 def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
2110 def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
2111 def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
2112 def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
2113 def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
2114 def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
2115 def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
2116 def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
2117 def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
2118 def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
2119 def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
2120 def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
2121 def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
2122 def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
2123 def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
2124 def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
2125 def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
2126 def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
2127 def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
2128 def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
2129 def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
2130 def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
2131 def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
2132 def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
2133 def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
2134 def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
2135 def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;
2136
2137 //
2138 // SVE2 - Non-widening pairwise arithmetic
2139 //
2140
2141 def int_aarch64_sve_addp    : AdvSIMD_Pred2VectorArg_Intrinsic;
2142 def int_aarch64_sve_faddp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2143 def int_aarch64_sve_fmaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2144 def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2145 def int_aarch64_sve_fminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2146 def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2147 def int_aarch64_sve_smaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2148 def int_aarch64_sve_sminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2149 def int_aarch64_sve_umaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2150 def int_aarch64_sve_uminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2151
2152 //
2153 // SVE2 - Widening pairwise arithmetic
2154 //
2155
2156 def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2157 def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2158
2159 //
2160 // SVE2 - Uniform complex integer arithmetic
2161 //
2162
2163 def int_aarch64_sve_cadd_x           : AdvSIMD_SVE2_CADD_Intrinsic;
2164 def int_aarch64_sve_sqcadd_x         : AdvSIMD_SVE2_CADD_Intrinsic;
2165 def int_aarch64_sve_cmla_x           : AdvSIMD_SVE2_CMLA_Intrinsic;
2166 def int_aarch64_sve_cmla_lane_x      : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2167 def int_aarch64_sve_sqrdcmlah_x      : AdvSIMD_SVE2_CMLA_Intrinsic;
2168 def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2169
2170 //
2171 // SVE2 - Widening complex integer arithmetic
2172 //
2173
2174 def int_aarch64_sve_saddlbt   : SVE2_2VectorArg_Long_Intrinsic;
2175 def int_aarch64_sve_ssublbt   : SVE2_2VectorArg_Long_Intrinsic;
2176 def int_aarch64_sve_ssubltb   : SVE2_2VectorArg_Long_Intrinsic;
2177
2178 //
2179 // SVE2 - Widening complex integer dot product
2180 //
2181
2182 def int_aarch64_sve_cdot      : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2183 def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;
2184
2185 //
2186 // SVE2 - Floating-point widening multiply-accumulate
2187 //
2188
2189 def int_aarch64_sve_fmlalb        : SVE2_3VectorArg_Long_Intrinsic;
2190 def int_aarch64_sve_fmlalb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2191 def int_aarch64_sve_fmlalt        : SVE2_3VectorArg_Long_Intrinsic;
2192 def int_aarch64_sve_fmlalt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2193 def int_aarch64_sve_fmlslb        : SVE2_3VectorArg_Long_Intrinsic;
2194 def int_aarch64_sve_fmlslb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2195 def int_aarch64_sve_fmlslt        : SVE2_3VectorArg_Long_Intrinsic;
2196 def int_aarch64_sve_fmlslt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2197
2198 //
2199 // SVE2 - Floating-point integer binary logarithm
2200 //
2201
2202 def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;
2203
2204 //
2205 // SVE2 - Vector histogram count
2206 //
2207
2208 def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
2209 def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;
2210
2211 //
2212 // SVE2 - Character match
2213 //
2214
2215 def int_aarch64_sve_match   : AdvSIMD_SVE_Compare_Intrinsic;
2216 def int_aarch64_sve_nmatch  : AdvSIMD_SVE_Compare_Intrinsic;
2217
2218 //
2219 // SVE2 - Unary narrowing operations
2220 //
2221
2222 def int_aarch64_sve_sqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2223 def int_aarch64_sve_sqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2224 def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
2225 def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2226 def int_aarch64_sve_uqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2227 def int_aarch64_sve_uqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2228
2229 //
2230 // SVE2 - Binary narrowing DSP operations
2231 //
2232 def int_aarch64_sve_addhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2233 def int_aarch64_sve_addhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2234
2235 def int_aarch64_sve_raddhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2236 def int_aarch64_sve_raddhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2237
2238 def int_aarch64_sve_subhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2239 def int_aarch64_sve_subhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2240
2241 def int_aarch64_sve_rsubhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2242 def int_aarch64_sve_rsubhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2243
2244 // Narrowing shift right
2245 def int_aarch64_sve_shrnb     : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2246 def int_aarch64_sve_shrnt     : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2247
2248 def int_aarch64_sve_rshrnb    : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2249 def int_aarch64_sve_rshrnt    : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2250
2251 // Saturating shift right - signed input/output
2252 def int_aarch64_sve_sqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2253 def int_aarch64_sve_sqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2254
2255 def int_aarch64_sve_sqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2256 def int_aarch64_sve_sqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2257
2258 // Saturating shift right - unsigned input/output
2259 def int_aarch64_sve_uqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2260 def int_aarch64_sve_uqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2261
2262 def int_aarch64_sve_uqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2263 def int_aarch64_sve_uqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2264
2265 // Saturating shift right - signed input, unsigned output
2266 def int_aarch64_sve_sqshrunb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2267 def int_aarch64_sve_sqshrunt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2268
2269 def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2270 def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2271
2272 // SVE2 MLA LANE.
2273 def int_aarch64_sve_smlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2274 def int_aarch64_sve_smlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2275 def int_aarch64_sve_umlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2276 def int_aarch64_sve_umlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2277 def int_aarch64_sve_smlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2278 def int_aarch64_sve_smlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2279 def int_aarch64_sve_umlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2280 def int_aarch64_sve_umlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2281 def int_aarch64_sve_smullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2282 def int_aarch64_sve_smullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2283 def int_aarch64_sve_umullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2284 def int_aarch64_sve_umullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2285 def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2286 def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2287 def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2288 def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2289 def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2290 def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2291
2292 // SVE2 MLA Unpredicated.
2293 def int_aarch64_sve_smlalb      : SVE2_3VectorArg_Long_Intrinsic;
2294 def int_aarch64_sve_smlalt      : SVE2_3VectorArg_Long_Intrinsic;
2295 def int_aarch64_sve_umlalb      : SVE2_3VectorArg_Long_Intrinsic;
2296 def int_aarch64_sve_umlalt      : SVE2_3VectorArg_Long_Intrinsic;
2297 def int_aarch64_sve_smlslb      : SVE2_3VectorArg_Long_Intrinsic;
2298 def int_aarch64_sve_smlslt      : SVE2_3VectorArg_Long_Intrinsic;
2299 def int_aarch64_sve_umlslb      : SVE2_3VectorArg_Long_Intrinsic;
2300 def int_aarch64_sve_umlslt      : SVE2_3VectorArg_Long_Intrinsic;
2301 def int_aarch64_sve_smullb      : SVE2_2VectorArg_Long_Intrinsic;
2302 def int_aarch64_sve_smullt      : SVE2_2VectorArg_Long_Intrinsic;
2303 def int_aarch64_sve_umullb      : SVE2_2VectorArg_Long_Intrinsic;
2304 def int_aarch64_sve_umullt      : SVE2_2VectorArg_Long_Intrinsic;
2305
2306 def int_aarch64_sve_sqdmlalb    : SVE2_3VectorArg_Long_Intrinsic;
2307 def int_aarch64_sve_sqdmlalt    : SVE2_3VectorArg_Long_Intrinsic;
2308 def int_aarch64_sve_sqdmlslb    : SVE2_3VectorArg_Long_Intrinsic;
2309 def int_aarch64_sve_sqdmlslt    : SVE2_3VectorArg_Long_Intrinsic;
2310 def int_aarch64_sve_sqdmullb    : SVE2_2VectorArg_Long_Intrinsic;
2311 def int_aarch64_sve_sqdmullt    : SVE2_2VectorArg_Long_Intrinsic;
2312 def int_aarch64_sve_sqdmlalbt   : SVE2_3VectorArg_Long_Intrinsic;
2313 def int_aarch64_sve_sqdmlslbt   : SVE2_3VectorArg_Long_Intrinsic;
2314
2315 // SVE2 ADDSUB Long Unpredicated.
2316 def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
2317 def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
2318 def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
2319 def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;
2320
2321 //
2322 // SVE2 - Polynomial arithmetic
2323 //
2324 def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
2325 def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
2326 def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
2327 def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
2328
2329 //
2330 // SVE2 bitwise ternary operations.
2331 //
2332 def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
2333 def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
2334 def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
2335 def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
2336 def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
2337 def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
2338 def int_aarch64_sve_xar    : AdvSIMD_2VectorArgIndexed_Intrinsic;
2339
2340 //
2341 // SVE2 - Optional AES, SHA-3 and SM4
2342 //
2343
2344 def int_aarch64_sve_aesd    : GCCBuiltin<"__builtin_sve_svaesd_u8">,
2345                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2346                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2347                                         [IntrNoMem]>;
2348 def int_aarch64_sve_aesimc  : GCCBuiltin<"__builtin_sve_svaesimc_u8">,
2349                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2350                                         [llvm_nxv16i8_ty],
2351                                         [IntrNoMem]>;
2352 def int_aarch64_sve_aese    : GCCBuiltin<"__builtin_sve_svaese_u8">,
2353                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2354                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2355                                         [IntrNoMem]>;
2356 def int_aarch64_sve_aesmc   : GCCBuiltin<"__builtin_sve_svaesmc_u8">,
2357                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2358                                         [llvm_nxv16i8_ty],
2359                                         [IntrNoMem]>;
2360 def int_aarch64_sve_rax1    : GCCBuiltin<"__builtin_sve_svrax1_u64">,
2361                               DefaultAttrsIntrinsic<[llvm_nxv2i64_ty],
2362                                         [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
2363                                         [IntrNoMem]>;
2364 def int_aarch64_sve_sm4e    : GCCBuiltin<"__builtin_sve_svsm4e_u32">,
2365                               DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
2366                                         [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2367                                         [IntrNoMem]>;
2368 def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">,
2369                               DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
2370                                         [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2371                                         [IntrNoMem]>;
2372 //
2373 // SVE2 - Extended table lookup/permute
2374 //
2375
2376 def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
2377 def int_aarch64_sve_tbx  : AdvSIMD_SVE2_TBX_Intrinsic;
2378
2379 //
2380 // SVE2 - Optional bit permutation
2381 //
2382
2383 def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
2384 def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
2385 def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;
2386
2387
2388 //
2389 // SVE ACLE: 7.3. INT8 matrix multiply extensions
2390 //
2391 def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
2392 def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
2393 def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;
2394
2395 def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
2396 def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2397 def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2398
2399 //
2400 // SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
2401 //
2402 def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
2403
2404 //
2405 // SVE ACLE: 7.2. BFloat16 extensions
2406 //
2407
2408 def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
2409 def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
2410 def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
2411
2412 def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;
2413
2414 def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
2415 def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
2416 def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
2417 }
2418
2419 //
2420 // SVE2 - Contiguous conflict detection
2421 //
2422
2423 def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2424 def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2425 def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2426 def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2427 def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2428 def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2429 def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2430 def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;