contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsAArch64.td

   1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines all of the AARCH64-specific intrinsics.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 let TargetPrefix = "aarch64" in {
  14
  15 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
  16                                  [IntrNoFree, IntrWillReturn]>;
  17 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
  18                                   [IntrNoFree, IntrWillReturn]>;
  19 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
  20                                  [IntrNoFree, IntrWillReturn]>;
  21 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
  22                                   [IntrNoFree, IntrWillReturn]>;
  23
  24 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
  25                                  [IntrNoFree, IntrWillReturn]>;
  26 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
  27                                   [IntrNoFree, IntrWillReturn]>;
  28 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
  29                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
  30                                [IntrNoFree, IntrWillReturn]>;
  31 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
  32                                   [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
  33                                   [IntrNoFree, IntrWillReturn]>;
  34
  35 def int_aarch64_clrex : Intrinsic<[]>;
  36
  37 def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  38                                 LLVMMatchType<0>], [IntrNoMem]>;
  39 def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  40                                 LLVMMatchType<0>], [IntrNoMem]>;
  41
  42 def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
  43
  44 def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
  45 def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
  46
  47 def int_aarch64_frint32z
  48     : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
  49                             [ IntrNoMem ]>;
  50 def int_aarch64_frint64z
  51     : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
  52                             [ IntrNoMem ]>;
  53 def int_aarch64_frint32x
  54     : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
  55                             [ IntrNoMem ]>;
  56 def int_aarch64_frint64x
  57     : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
  58                             [ IntrNoMem ]>;
  59
  60 //===----------------------------------------------------------------------===//
  61 // HINT
  62
  63 def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;
  64
  65 def int_aarch64_break : Intrinsic<[], [llvm_i32_ty],
  66     [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>;
  67
  68 //===----------------------------------------------------------------------===//
  69 // Data Barrier Instructions
  70
  71 def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">,
  72                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
  73 def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">,
  74                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
  75 def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">,
  76                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
  77
  78 // A space-consuming intrinsic primarily for testing block and jump table
  79 // placements. The first argument is the number of bytes this "instruction"
  80 // takes up, the second and return value are essentially chains, used to force
  81 // ordering during ISel.
  82 def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
  83
  84 }
  85
  86 //===----------------------------------------------------------------------===//
  87 // Advanced SIMD (NEON)
  88
  89 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  90   class AdvSIMD_2Scalar_Float_Intrinsic
  91     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  92                 [IntrNoMem]>;
  93
  94   class AdvSIMD_FPToIntRounding_Intrinsic
  95     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
  96
  97   class AdvSIMD_1IntArg_Intrinsic
  98     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  99   class AdvSIMD_1FloatArg_Intrinsic
 100     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 101   class AdvSIMD_1VectorArg_Intrinsic
 102     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 103   class AdvSIMD_1VectorArg_Expand_Intrinsic
 104     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
 105   class AdvSIMD_1VectorArg_Long_Intrinsic
 106     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
 107   class AdvSIMD_1IntArg_Narrow_Intrinsic
 108     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
 109   class AdvSIMD_1VectorArg_Narrow_Intrinsic
 110     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
 111   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
 112     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
 113   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
 114     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
 115
 116   class AdvSIMD_2IntArg_Intrinsic
 117     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
 118                 [IntrNoMem]>;
 119   class AdvSIMD_2FloatArg_Intrinsic
 120     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
 121                 [IntrNoMem]>;
 122   class AdvSIMD_2VectorArg_Intrinsic
 123     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
 124                 [IntrNoMem]>;
 125   class AdvSIMD_2VectorArg_Compare_Intrinsic
 126     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
 127                 [IntrNoMem]>;
 128   class AdvSIMD_2Arg_FloatCompare_Intrinsic
 129     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
 130                 [IntrNoMem]>;
 131   class AdvSIMD_2VectorArg_Long_Intrinsic
 132     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 133                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
 134                 [IntrNoMem]>;
 135   class AdvSIMD_2VectorArg_Wide_Intrinsic
 136     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 137                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
 138                 [IntrNoMem]>;
 139   class AdvSIMD_2VectorArg_Narrow_Intrinsic
 140     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 141                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
 142                 [IntrNoMem]>;
 143   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
 144     : DefaultAttrsIntrinsic<[llvm_anyint_ty],
 145                 [LLVMExtendedType<0>, llvm_i32_ty],
 146                 [IntrNoMem]>;
 147   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
 148     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 149                 [llvm_anyvector_ty],
 150                 [IntrNoMem]>;
 151   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
 152     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 153                 [LLVMTruncatedType<0>],
 154                 [IntrNoMem]>;
 155   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
 156     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 157                 [LLVMTruncatedType<0>, llvm_i32_ty],
 158                 [IntrNoMem]>;
 159   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
 160     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 161                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
 162                 [IntrNoMem]>;
 163   class AdvSIMD_2VectorArg_Lane_Intrinsic
 164     : DefaultAttrsIntrinsic<[llvm_anyint_ty],
 165                 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
 166                 [IntrNoMem]>;
 167
 168   class AdvSIMD_3IntArg_Intrinsic
 169     : DefaultAttrsIntrinsic<[llvm_anyint_ty],
 170                 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 171                 [IntrNoMem]>;
 172   class AdvSIMD_3VectorArg_Intrinsic
 173       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 174                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 175                [IntrNoMem]>;
 176   class AdvSIMD_3VectorArg_Scalar_Intrinsic
 177       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 178                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
 179                [IntrNoMem]>;
 180   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
 181       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 182                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
 183                 LLVMMatchType<1>], [IntrNoMem]>;
 184   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
 185     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 186                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
 187                 [IntrNoMem]>;
 188   class AdvSIMD_CvtFxToFP_Intrinsic
 189     : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
 190                 [IntrNoMem]>;
 191   class AdvSIMD_CvtFPToFx_Intrinsic
 192     : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
 193                 [IntrNoMem]>;
 194
 195   class AdvSIMD_1Arg_Intrinsic
 196     : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 197
 198   class AdvSIMD_Dot_Intrinsic
 199     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 200                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 201                 [IntrNoMem]>;
 202
 203   class AdvSIMD_FP16FML_Intrinsic
 204     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 205                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 206                 [IntrNoMem]>;
 207
 208   class AdvSIMD_MatMul_Intrinsic
 209     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 210                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 211                 [IntrNoMem]>;
 212
 213   class AdvSIMD_FML_Intrinsic
 214     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 215                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 216                 [IntrNoMem]>;
 217
 218   class AdvSIMD_BF16FML_Intrinsic
 219     : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
 220                 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
 221                 [IntrNoMem]>;
 222 }
 223
 224 // Arithmetic ops
 225
 226 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
 227   // Vector Add Across Lanes
 228   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 229   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 230   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 231
 232   // Vector Long Add Across Lanes
 233   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 234   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 235
 236   // Vector Halving Add
 237   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
 238   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
 239
 240   // Vector Rounding Halving Add
 241   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
 242   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
 243
 244   // Vector Saturating Add
 245   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
 246   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
 247   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
 248   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
 249
 250   // Vector Add High-Half
 251   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 252   // header is no longer supported.
 253   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 254
 255   // Vector Rounding Add High-Half
 256   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 257
 258   // Vector Saturating Doubling Multiply High
 259   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
 260   def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
 261   def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
 262
 263   // Vector Saturating Rounding Doubling Multiply High
 264   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
 265   def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
 266   def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
 267
 268   def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic;
 269   def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic;
 270
 271   // Vector Polynominal Multiply
 272   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
 273
 274   // Vector Long Multiply
 275   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
 276   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
 277   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 278
 279   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
 280   // it with a v16i8.
 281   def int_aarch64_neon_pmull64 :
 282         DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 283
 284   // Vector Extending Multiply
 285   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
 286     let IntrProperties = [IntrNoMem, Commutative];
 287   }
 288
 289   // Vector Saturating Doubling Long Multiply
 290   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 291   def int_aarch64_neon_sqdmulls_scalar
 292     : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 293
 294   // Vector Halving Subtract
 295   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
 296   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
 297
 298   // Vector Saturating Subtract
 299   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
 300   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
 301
 302   // Vector Subtract High-Half
 303   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 304   // header is no longer supported.
 305   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 306
 307   // Vector Rounding Subtract High-Half
 308   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 309
 310   // Vector Compare Absolute Greater-than-or-equal
 311   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 312
 313   // Vector Compare Absolute Greater-than
 314   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 315
 316   // Vector Absolute Difference
 317   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
 318   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
 319   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
 320
 321   // Scalar Absolute Difference
 322   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
 323
 324   // Vector Max
 325   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
 326   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
 327   def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
 328   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
 329
 330   // Vector Max Across Lanes
 331   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 332   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 333   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 334   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 335
 336   // Vector Min
 337   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
 338   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
 339   def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
 340   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
 341
 342   // Vector Min/Max Number
 343   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
 344   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
 345
 346   // Vector Min Across Lanes
 347   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 348   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 349   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 350   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 351
 352   // Pairwise Add
 353   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
 354   def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
 355
 356   // Long Pairwise Add
 357   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
 358   // uaddlp, but tblgen's type inference currently can't handle the
 359   // pattern fragments this ends up generating.
 360   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 361   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 362
 363   // Folding Maximum
 364   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
 365   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
 366   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
 367
 368   // Folding Minimum
 369   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
 370   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
 371   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
 372
 373   // Reciprocal Estimate/Step
 374   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
 375   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
 376
 377   // Reciprocal Exponent
 378   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
 379
 380   // Vector Saturating Shift Left
 381   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
 382   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
 383
 384   // Vector Rounding Shift Left
 385   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
 386   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
 387
 388   // Vector Saturating Rounding Shift Left
 389   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
 390   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
 391
 392   // Vector Signed->Unsigned Shift Left by Constant
 393   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
 394
 395   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
 396   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 397
 398   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
 399   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 400
 401   // Vector Narrowing Shift Right by Constant
 402   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 403   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 404
 405   // Vector Rounding Narrowing Shift Right by Constant
 406   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 407
 408   // Vector Rounding Narrowing Saturating Shift Right by Constant
 409   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 410   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 411
 412   // Vector Shift Left
 413   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
 414   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
 415
 416   // Vector Widening Shift Left by Constant
 417   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
 418   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 419   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 420
 421   // Vector Shift Right by Constant and Insert
 422   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 423
 424   // Vector Shift Left by Constant and Insert
 425   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 426
 427   // Vector Saturating Narrow
 428   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
 429   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
 430   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 431   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 432
 433   // Vector Saturating Extract and Unsigned Narrow
 434   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
 435   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 436
 437   // Vector Absolute Value
 438   def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
 439
 440   // Vector Saturating Absolute Value
 441   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
 442
 443   // Vector Saturating Negation
 444   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
 445
 446   // Vector Count Leading Sign Bits
 447   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
 448
 449   // Vector Reciprocal Estimate
 450   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
 451   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
 452
 453   // Vector Square Root Estimate
 454   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
 455   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
 456
 457   // Vector Conversions Between Half-Precision and Single-Precision.
 458   def int_aarch64_neon_vcvtfp2hf
 459     : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 460   def int_aarch64_neon_vcvthf2fp
 461     : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
 462
 463   // Vector Conversions Between Floating-point and Fixed-point.
 464   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
 465   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
 466   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 467   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 468
 469   // Vector FP->Int Conversions
 470   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
 471   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
 472   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
 473   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
 474   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
 475   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
 476   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
 477   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
 478   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
 479   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 480
 481   // v8.5-A Vector FP Rounding
 482   def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic;
 483   def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic;
 484   def int_aarch64_neon_frint64x : AdvSIMD_1FloatArg_Intrinsic;
 485   def int_aarch64_neon_frint64z : AdvSIMD_1FloatArg_Intrinsic;
 486
 487   // Scalar FP->Int conversions
 488
 489   // Vector FP Inexact Narrowing
 490   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
 491
 492   // Scalar FP Inexact Narrowing
 493   def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty],
 494                                         [IntrNoMem]>;
 495
 496   // v8.2-A Dot Product
 497   def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
 498   def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
 499
 500   // v8.6-A Matrix Multiply Intrinsics
 501   def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
 502   def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
 503   def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
 504   def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
 505   def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
 506   def int_aarch64_neon_bfmmla
 507     : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
 508                 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
 509                 [IntrNoMem]>;
 510   def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic;
 511   def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic;
 512
 513
 514   // v8.6-A Bfloat Intrinsics
 515   def int_aarch64_neon_bfcvt
 516     : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
 517   def int_aarch64_neon_bfcvtn
 518     : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 519   def int_aarch64_neon_bfcvtn2
 520     : DefaultAttrsIntrinsic<[llvm_v8bf16_ty],
 521                 [llvm_v8bf16_ty, llvm_v4f32_ty],
 522                 [IntrNoMem]>;
 523
 524   // v8.2-A FP16 Fused Multiply-Add Long
 525   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
 526   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
 527   def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
 528   def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
 529
 530   // v8.3-A Floating-point complex add
 531   def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
 532   def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
 533
 534   def int_aarch64_neon_vcmla_rot0   : AdvSIMD_3VectorArg_Intrinsic;
 535   def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;
 536   def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;
 537   def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;
 538 }
 539
 540 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 541   class AdvSIMD_2Vector2Index_Intrinsic
 542     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 543                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
 544                 [IntrNoMem]>;
 545 }
 546
 547 // Vector element to element moves
 548 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
 549
 550 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 551   class AdvSIMD_1Vec_Load_Intrinsic
 552       : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
 553                   [IntrReadMem, IntrArgMemOnly]>;
 554   class AdvSIMD_1Vec_Store_Lane_Intrinsic
 555     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
 556                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 557
 558   class AdvSIMD_2Vec_Load_Intrinsic
 559     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
 560                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 561                 [IntrReadMem, IntrArgMemOnly]>;
 562   class AdvSIMD_2Vec_Load_Lane_Intrinsic
 563     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
 564                 [LLVMMatchType<0>, llvm_anyvector_ty,
 565                  llvm_i64_ty, llvm_anyptr_ty],
 566                 [IntrReadMem, IntrArgMemOnly]>;
 567   class AdvSIMD_2Vec_Store_Intrinsic
 568     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 569                      LLVMAnyPointerType<LLVMMatchType<0>>],
 570                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 571   class AdvSIMD_2Vec_Store_Lane_Intrinsic
 572     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 573                  llvm_i64_ty, llvm_anyptr_ty],
 574                 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 575
 576   class AdvSIMD_3Vec_Load_Intrinsic
 577     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
 578                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 579                 [IntrReadMem, IntrArgMemOnly]>;
 580   class AdvSIMD_3Vec_Load_Lane_Intrinsic
 581     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 582                 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
 583                  llvm_i64_ty, llvm_anyptr_ty],
 584                 [IntrReadMem, IntrArgMemOnly]>;
 585   class AdvSIMD_3Vec_Store_Intrinsic
 586     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 587                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
 588                 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 589   class AdvSIMD_3Vec_Store_Lane_Intrinsic
 590     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,
 591                  LLVMMatchType<0>, LLVMMatchType<0>,
 592                  llvm_i64_ty, llvm_anyptr_ty],
 593                 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 594
 595   class AdvSIMD_4Vec_Load_Intrinsic
 596     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 597                  LLVMMatchType<0>, llvm_anyvector_ty],
 598                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 599                 [IntrReadMem, IntrArgMemOnly]>;
 600   class AdvSIMD_4Vec_Load_Lane_Intrinsic
 601     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 602                  LLVMMatchType<0>, LLVMMatchType<0>],
 603                 [LLVMMatchType<0>, LLVMMatchType<0>,
 604                  LLVMMatchType<0>, llvm_anyvector_ty,
 605                  llvm_i64_ty, llvm_anyptr_ty],
 606                 [IntrReadMem, IntrArgMemOnly]>;
 607   class AdvSIMD_4Vec_Store_Intrinsic
 608     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 609                  LLVMMatchType<0>, LLVMMatchType<0>,
 610                  LLVMAnyPointerType<LLVMMatchType<0>>],
 611                 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 612   class AdvSIMD_4Vec_Store_Lane_Intrinsic
 613     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 614                  LLVMMatchType<0>, LLVMMatchType<0>,
 615                  llvm_i64_ty, llvm_anyptr_ty],
 616                 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 617 }
 618
 619 // Memory ops
 620
 621 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
 622 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
 623 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
 624
 625 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
 626 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
 627 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
 628
 629 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
 630 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
 631 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
 632
 633 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
 634 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
 635 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
 636
 637 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
 638 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
 639 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
 640
 641 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
 642 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
 643 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
 644
 645 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
 646 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
 647 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
 648
 649 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 650   class AdvSIMD_Tbl1_Intrinsic
 651     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
 652                 [IntrNoMem]>;
 653   class AdvSIMD_Tbl2_Intrinsic
 654     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 655                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
 656   class AdvSIMD_Tbl3_Intrinsic
 657     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 658                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 659                  LLVMMatchType<0>],
 660                 [IntrNoMem]>;
 661   class AdvSIMD_Tbl4_Intrinsic
 662     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 663                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 664                  LLVMMatchType<0>],
 665                 [IntrNoMem]>;
 666
 667   class AdvSIMD_Tbx1_Intrinsic
 668     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 669                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
 670                 [IntrNoMem]>;
 671   class AdvSIMD_Tbx2_Intrinsic
 672     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 673                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 674                  LLVMMatchType<0>],
 675                 [IntrNoMem]>;
 676   class AdvSIMD_Tbx3_Intrinsic
 677     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 678                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 679                  llvm_v16i8_ty, LLVMMatchType<0>],
 680                 [IntrNoMem]>;
 681   class AdvSIMD_Tbx4_Intrinsic
 682     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 683                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 684                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
 685                 [IntrNoMem]>;
 686 }
 687 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
 688 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
 689 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
 690 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
 691
 692 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 693 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 694 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 695 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 696
 697 let TargetPrefix = "aarch64" in {
 698   class FPCR_Get_Intrinsic
 699     : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
 700   class FPCR_Set_Intrinsic
 701     : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>;
 702   class RNDR_Intrinsic
 703     : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>;
 704 }
 705
 706 // FPCR
 707 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
 708 def int_aarch64_set_fpcr : FPCR_Set_Intrinsic;
 709
 710 // Armv8.5-A Random number generation intrinsics
 711 def int_aarch64_rndr : RNDR_Intrinsic;
 712 def int_aarch64_rndrrs : RNDR_Intrinsic;
 713
 714 let TargetPrefix = "aarch64" in {
 715   class Crypto_AES_DataKey_Intrinsic
 716     : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 717
 718   class Crypto_AES_Data_Intrinsic
 719     : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 720
 721   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 722   // (v4i32).
 723   class Crypto_SHA_5Hash4Schedule_Intrinsic
 724     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
 725                 [IntrNoMem]>;
 726
 727   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 728   // (v4i32).
 729   class Crypto_SHA_1Hash_Intrinsic
 730     : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 731
 732   // SHA intrinsic taking 8 words of the schedule
 733   class Crypto_SHA_8Schedule_Intrinsic
 734     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 735
 736   // SHA intrinsic taking 12 words of the schedule
 737   class Crypto_SHA_12Schedule_Intrinsic
 738     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 739                 [IntrNoMem]>;
 740
 741   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
 742   class Crypto_SHA_8Hash4Schedule_Intrinsic
 743     : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 744                 [IntrNoMem]>;
 745
 746   // SHA512 intrinsic taking 2 arguments
 747   class Crypto_SHA512_2Arg_Intrinsic
 748     : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>;
 749
 750   // SHA512 intrinsic taking 3 Arguments
 751   class Crypto_SHA512_3Arg_Intrinsic
 752     : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty],
 753                 [IntrNoMem]>;
 754
 755   // SHA3 Intrinsics taking 3 arguments
 756   class Crypto_SHA3_3Arg_Intrinsic
 757     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 758                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 759                [IntrNoMem]>;
 760
 761   // SHA3 Intrinsic taking 2 arguments
 762   class Crypto_SHA3_2Arg_Intrinsic
 763     : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
 764                [IntrNoMem]>;
 765
 766   // SHA3 Intrinsic taking 3 Arguments 1 immediate
 767   class Crypto_SHA3_2ArgImm_Intrinsic
 768     : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i64_ty],
 769                [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 770
 771   class Crypto_SM3_3Vector_Intrinsic
 772     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 773                 [IntrNoMem]>;
 774
 775   class Crypto_SM3_3VectorIndexed_Intrinsic
 776     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i64_ty],
 777                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
 778
 779   class Crypto_SM4_2Vector_Intrinsic
 780     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 781 }
 782
 783 // AES
 784 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
 785 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
 786 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
 787 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
 788
 789 // SHA1
 790 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 791 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 792 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 793 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
 794
 795 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
 796 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
 797
 798 // SHA256
 799 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
 800 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
 801 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
 802 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
 803
 804 //SHA3
 805 def int_aarch64_crypto_eor3s : Crypto_SHA3_3Arg_Intrinsic;
 806 def int_aarch64_crypto_eor3u : Crypto_SHA3_3Arg_Intrinsic;
 807 def int_aarch64_crypto_bcaxs : Crypto_SHA3_3Arg_Intrinsic;
 808 def int_aarch64_crypto_bcaxu : Crypto_SHA3_3Arg_Intrinsic;
 809 def int_aarch64_crypto_rax1 : Crypto_SHA3_2Arg_Intrinsic;
 810 def int_aarch64_crypto_xar : Crypto_SHA3_2ArgImm_Intrinsic;
 811
 812 // SHA512
 813 def int_aarch64_crypto_sha512h : Crypto_SHA512_3Arg_Intrinsic;
 814 def int_aarch64_crypto_sha512h2 : Crypto_SHA512_3Arg_Intrinsic;
 815 def int_aarch64_crypto_sha512su0 : Crypto_SHA512_2Arg_Intrinsic;
 816 def int_aarch64_crypto_sha512su1 : Crypto_SHA512_3Arg_Intrinsic;
 817
 818 //SM3 & SM4
 819 def int_aarch64_crypto_sm3partw1 : Crypto_SM3_3Vector_Intrinsic;
 820 def int_aarch64_crypto_sm3partw2 : Crypto_SM3_3Vector_Intrinsic;
 821 def int_aarch64_crypto_sm3ss1    : Crypto_SM3_3Vector_Intrinsic;
 822 def int_aarch64_crypto_sm3tt1a   : Crypto_SM3_3VectorIndexed_Intrinsic;
 823 def int_aarch64_crypto_sm3tt1b   : Crypto_SM3_3VectorIndexed_Intrinsic;
 824 def int_aarch64_crypto_sm3tt2a   : Crypto_SM3_3VectorIndexed_Intrinsic;
 825 def int_aarch64_crypto_sm3tt2b   : Crypto_SM3_3VectorIndexed_Intrinsic;
 826 def int_aarch64_crypto_sm4e      : Crypto_SM4_2Vector_Intrinsic;
 827 def int_aarch64_crypto_sm4ekey   : Crypto_SM4_2Vector_Intrinsic;
 828
 829 //===----------------------------------------------------------------------===//
 830 // CRC32
 831
 832 let TargetPrefix = "aarch64" in {
 833
 834 def int_aarch64_crc32b  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 835     [IntrNoMem]>;
 836 def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 837     [IntrNoMem]>;
 838 def int_aarch64_crc32h  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 839     [IntrNoMem]>;
 840 def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 841     [IntrNoMem]>;
 842 def int_aarch64_crc32w  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 843     [IntrNoMem]>;
 844 def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 845     [IntrNoMem]>;
 846 def int_aarch64_crc32x  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 847     [IntrNoMem]>;
 848 def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 849     [IntrNoMem]>;
 850 }
 851
 852 //===----------------------------------------------------------------------===//
 853 // Memory Tagging Extensions (MTE) Intrinsics
 854 let TargetPrefix = "aarch64" in {
 855 def int_aarch64_irg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 856     [IntrNoMem, IntrHasSideEffects]>;
 857 def int_aarch64_addg  : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 858     [IntrNoMem]>;
 859 def int_aarch64_gmi   : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
 860     [IntrNoMem]>;
 861 def int_aarch64_ldg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
 862     [IntrReadMem]>;
 863 def int_aarch64_stg   : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
 864     [IntrWriteMem]>;
 865 def int_aarch64_subp :  DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
 866     [IntrNoMem]>;
 867
 868 // The following are codegen-only intrinsics for stack instrumentation.
 869
 870 // Generate a randomly tagged stack base pointer.
 871 def int_aarch64_irg_sp   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty],
 872     [IntrNoMem, IntrHasSideEffects]>;
 873
 874 // Transfer pointer tag with offset.
 875 // ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
 876 // * address is the address in ptr0
 877 // * tag is a function of (tag in baseptr, tag_offset).
 878 // ** Beware, this is not the same function as implemented by the ADDG instruction!
 879 //    Backend optimizations may change tag_offset; the only guarantee is that calls
 880 //    to tagp with the same pair of (baseptr, tag_offset) will produce pointers
 881 //    with the same tag value, assuming the set of excluded tags has not changed.
 882 // Address bits in baseptr and tag bits in ptr0 are ignored.
 883 // When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
 884 //   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
 885 // It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
 886 def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
 887     [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 888
 889 // Update allocation tags for the memory range to match the tag in the pointer argument.
 890 def int_aarch64_settag  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 891     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 892
 893 // Update allocation tags for the memory range to match the tag in the pointer argument,
 894 // and set memory contents to zero.
 895 def int_aarch64_settag_zero  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 896     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 897
 898 // Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
 899 def int_aarch64_stgp  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
 900     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 901 }
 902
 903 //===----------------------------------------------------------------------===//
 904 // Memory Operations (MOPS) Intrinsics
 905 let TargetPrefix = "aarch64" in {
 906   // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64
 907   def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty],
 908       [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 909 }
 910
 911 // Transactional Memory Extension (TME) Intrinsics
 912 let TargetPrefix = "aarch64" in {
 913 def int_aarch64_tstart  : ClangBuiltin<"__builtin_arm_tstart">,
 914                          Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;
 915
 916 def int_aarch64_tcommit : ClangBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>;
 917
 918 def int_aarch64_tcancel : ClangBuiltin<"__builtin_arm_tcancel">,
 919                           Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>;
 920
 921 def int_aarch64_ttest   : ClangBuiltin<"__builtin_arm_ttest">,
 922                           Intrinsic<[llvm_i64_ty], [],
 923                                     [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
 924
 925 // Armv8.7-A load/store 64-byte intrinsics
 926 defvar data512 = !listsplat(llvm_i64_ty, 8);
 927 def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>;
 928 def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;
 929 def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
 930 def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
 931
 932 }
 933
 934 def llvm_nxv1i1_ty  : LLVMType<nxv1i1>;
 935 def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
 936 def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
 937 def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
 938 def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
 939 def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
 940 def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
 941 def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
 942 def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
 943 def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
 944 def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
 945 def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
 946
 947 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 948
 949   class AdvSIMD_SVE_Create_2Vector_Tuple
 950     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 951                 [llvm_anyvector_ty, LLVMMatchType<1>],
 952                 [IntrReadMem]>;
 953
 954   class AdvSIMD_SVE_Create_3Vector_Tuple
 955     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 956                 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
 957                 [IntrReadMem]>;
 958
 959   class AdvSIMD_SVE_Create_4Vector_Tuple
 960     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 961                 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
 962                  LLVMMatchType<1>],
 963                 [IntrReadMem]>;
 964
 965   class AdvSIMD_SVE_Set_Vector_Tuple
 966     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 967                 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
 968                 [IntrReadMem, ImmArg<ArgIndex<1>>]>;
 969
 970   class AdvSIMD_SVE_Get_Vector_Tuple
 971     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
 972                 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 973
 974   class AdvSIMD_ManyVec_PredLoad_Intrinsic
 975     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
 976                 [IntrReadMem, IntrArgMemOnly]>;
 977
 978   class AdvSIMD_1Vec_PredLoad_Intrinsic
 979     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 980                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 981                  LLVMPointerToElt<0>],
 982                 [IntrReadMem, IntrArgMemOnly]>;
 983
 984   class AdvSIMD_2Vec_PredLoad_Intrinsic
 985     : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
 986                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 987                  LLVMPointerToElt<0>],
 988                 [IntrReadMem, IntrArgMemOnly]>;
 989
 990   class AdvSIMD_3Vec_PredLoad_Intrinsic
 991     : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
 992                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 993                  LLVMPointerToElt<0>],
 994                 [IntrReadMem, IntrArgMemOnly]>;
 995
 996   class AdvSIMD_4Vec_PredLoad_Intrinsic
 997     : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
 998                  LLVMMatchType<0>],
 999                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1000                  LLVMPointerToElt<0>],
1001                 [IntrReadMem, IntrArgMemOnly]>;
1002
1003   class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic
1004     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1005                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1006                  LLVMPointerToElt<0>],
1007                 [IntrInaccessibleMemOrArgMemOnly]>;
1008
1009   class AdvSIMD_1Vec_PredStore_Intrinsic
1010     : DefaultAttrsIntrinsic<[],
1011                 [llvm_anyvector_ty,
1012                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1013                  LLVMPointerToElt<0>],
1014                 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
1015
1016   class AdvSIMD_2Vec_PredStore_Intrinsic
1017       : DefaultAttrsIntrinsic<[],
1018                   [llvm_anyvector_ty, LLVMMatchType<0>,
1019                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
1020                   [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
1021
1022   class AdvSIMD_3Vec_PredStore_Intrinsic
1023       : DefaultAttrsIntrinsic<[],
1024                   [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
1025                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
1026                   [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
1027
1028   class AdvSIMD_4Vec_PredStore_Intrinsic
1029       : DefaultAttrsIntrinsic<[],
1030                   [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
1031                    LLVMMatchType<0>,
1032                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
1033                   [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
1034
1035   class AdvSIMD_SVE_Index_Intrinsic
1036     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1037                 [LLVMVectorElementType<0>,
1038                  LLVMVectorElementType<0>],
1039                 [IntrNoMem]>;
1040
1041   class AdvSIMD_Merged1VectorArg_Intrinsic
1042     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1043                 [LLVMMatchType<0>,
1044                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1045                  LLVMMatchType<0>],
1046                 [IntrNoMem]>;
1047
1048   class AdvSIMD_2VectorArgIndexed_Intrinsic
1049     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1050                 [LLVMMatchType<0>,
1051                  LLVMMatchType<0>,
1052                  llvm_i32_ty],
1053                 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1054
1055   class AdvSIMD_3VectorArgIndexed_Intrinsic
1056     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1057                 [LLVMMatchType<0>,
1058                  LLVMMatchType<0>,
1059                  LLVMMatchType<0>,
1060                  llvm_i32_ty],
1061                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1062
1063   class AdvSIMD_Pred1VectorArg_Intrinsic
1064     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1065                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1066                  LLVMMatchType<0>],
1067                 [IntrNoMem]>;
1068
1069   class AdvSIMD_Pred2VectorArg_Intrinsic
1070     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1071                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1072                  LLVMMatchType<0>,
1073                  LLVMMatchType<0>],
1074                 [IntrNoMem]>;
1075
1076   class AdvSIMD_Pred3VectorArg_Intrinsic
1077     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1078                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1079                  LLVMMatchType<0>,
1080                  LLVMMatchType<0>,
1081                  LLVMMatchType<0>],
1082                 [IntrNoMem]>;
1083
1084   class AdvSIMD_SVE_Compare_Intrinsic
1085     : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
1086                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1087                  llvm_anyvector_ty,
1088                  LLVMMatchType<0>],
1089                 [IntrNoMem]>;
1090
1091   class AdvSIMD_SVE_CompareWide_Intrinsic
1092     : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
1093                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1094                  llvm_anyvector_ty,
1095                  llvm_nxv2i64_ty],
1096                 [IntrNoMem]>;
1097
1098   class AdvSIMD_SVE_Saturating_Intrinsic
1099     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1100                 [LLVMMatchType<0>,
1101                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
1102                 [IntrNoMem]>;
1103
1104   class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
1105     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1106                 [LLVMMatchType<0>,
1107                  llvm_i32_ty,
1108                  llvm_i32_ty],
1109                 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
1110
1111   class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
1112     : DefaultAttrsIntrinsic<[T],
1113                 [T, llvm_anyvector_ty],
1114                 [IntrNoMem]>;
1115
1116   class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
1117     : DefaultAttrsIntrinsic<[T],
1118                 [T, llvm_i32_ty, llvm_i32_ty],
1119                 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
1120
1121   class AdvSIMD_SVE_CNT_Intrinsic
1122     : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>],
1123                 [LLVMVectorOfBitcastsToInt<0>,
1124                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1125                  llvm_anyvector_ty],
1126                 [IntrNoMem]>;
1127
1128   class AdvSIMD_SVE_ReduceWithInit_Intrinsic
1129     : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
1130                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1131                  LLVMVectorElementType<0>,
1132                  llvm_anyvector_ty],
1133                 [IntrNoMem]>;
1134
1135   class AdvSIMD_SVE_ShiftByImm_Intrinsic
1136     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1137                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1138                  LLVMMatchType<0>,
1139                  llvm_i32_ty],
1140                 [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1141
1142   class AdvSIMD_SVE_ShiftWide_Intrinsic
1143     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1144                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1145                  LLVMMatchType<0>,
1146                  llvm_nxv2i64_ty],
1147                 [IntrNoMem]>;
1148
1149   class AdvSIMD_SVE_Unpack_Intrinsic
1150     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1151                [LLVMSubdivide2VectorType<0>],
1152                [IntrNoMem]>;
1153
1154   class AdvSIMD_SVE_CADD_Intrinsic
1155     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1156                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1157                  LLVMMatchType<0>,
1158                  LLVMMatchType<0>,
1159                  llvm_i32_ty],
1160                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1161
1162   class AdvSIMD_SVE_CMLA_Intrinsic
1163     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1164                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1165                  LLVMMatchType<0>,
1166                  LLVMMatchType<0>,
1167                  LLVMMatchType<0>,
1168                  llvm_i32_ty],
1169                 [IntrNoMem, ImmArg<ArgIndex<4>>]>;
1170
1171   class AdvSIMD_SVE_CMLA_LANE_Intrinsic
1172     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1173                 [LLVMMatchType<0>,
1174                  LLVMMatchType<0>,
1175                  LLVMMatchType<0>,
1176                  llvm_i32_ty,
1177                  llvm_i32_ty],
1178                 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1179
1180   class AdvSIMD_SVE_DUP_Intrinsic
1181     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1182                 [LLVMMatchType<0>,
1183                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1184                  LLVMVectorElementType<0>],
1185                 [IntrNoMem]>;
1186
1187   class AdvSIMD_SVE_DUP_Unpred_Intrinsic
1188     : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
1189                 [IntrNoMem]>;
1190
1191   class AdvSIMD_SVE_DUPQ_Intrinsic
1192     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1193                 [LLVMMatchType<0>,
1194                  llvm_i64_ty],
1195                 [IntrNoMem]>;
1196
1197   class AdvSIMD_SVE_EXPA_Intrinsic
1198     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1199                 [LLVMVectorOfBitcastsToInt<0>],
1200                 [IntrNoMem]>;
1201
1202   class AdvSIMD_SVE_FCVT_Intrinsic
1203     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1204                 [LLVMMatchType<0>,
1205                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1206                  llvm_anyvector_ty],
1207                 [IntrNoMem]>;
1208
1209   class AdvSIMD_SVE_FCVTZS_Intrinsic
1210     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1211                 [LLVMVectorOfBitcastsToInt<0>,
1212                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1213                  llvm_anyvector_ty],
1214                 [IntrNoMem]>;
1215
1216   class AdvSIMD_SVE_INSR_Intrinsic
1217     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1218                 [LLVMMatchType<0>,
1219                  LLVMVectorElementType<0>],
1220                 [IntrNoMem]>;
1221
1222   class AdvSIMD_SVE_PTRUE_Intrinsic
1223     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1224                 [llvm_i32_ty],
1225                 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1226
1227   class AdvSIMD_SVE_PUNPKHI_Intrinsic
1228     : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>],
1229                 [llvm_anyvector_ty],
1230                 [IntrNoMem]>;
1231
1232   class AdvSIMD_SVE_SCALE_Intrinsic
1233     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1234                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1235                  LLVMMatchType<0>,
1236                  LLVMVectorOfBitcastsToInt<0>],
1237                 [IntrNoMem]>;
1238
1239   class AdvSIMD_SVE_SCVTF_Intrinsic
1240     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1241                 [LLVMMatchType<0>,
1242                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1243                  llvm_anyvector_ty],
1244                 [IntrNoMem]>;
1245
1246   class AdvSIMD_SVE_TSMUL_Intrinsic
1247     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1248                 [LLVMMatchType<0>,
1249                  LLVMVectorOfBitcastsToInt<0>],
1250                 [IntrNoMem]>;
1251
1252   class AdvSIMD_SVE_CNTB_Intrinsic
1253     : DefaultAttrsIntrinsic<[llvm_i64_ty],
1254                 [llvm_i32_ty],
1255                 [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1256
1257   class AdvSIMD_SVE_CNTP_Intrinsic
1258     : DefaultAttrsIntrinsic<[llvm_i64_ty],
1259                 [llvm_anyvector_ty, LLVMMatchType<0>],
1260                 [IntrNoMem]>;
1261
1262   class AdvSIMD_SVE_DOT_Intrinsic
1263     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1264                 [LLVMMatchType<0>,
1265                  LLVMSubdivide4VectorType<0>,
1266                  LLVMSubdivide4VectorType<0>],
1267                 [IntrNoMem]>;
1268
1269   class AdvSIMD_SVE_DOT_Indexed_Intrinsic
1270     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1271                 [LLVMMatchType<0>,
1272                  LLVMSubdivide4VectorType<0>,
1273                  LLVMSubdivide4VectorType<0>,
1274                  llvm_i32_ty],
1275                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1276
1277   class AdvSIMD_SVE_PTEST_Intrinsic
1278     : DefaultAttrsIntrinsic<[llvm_i1_ty],
1279                 [llvm_anyvector_ty,
1280                  LLVMMatchType<0>],
1281                 [IntrNoMem]>;
1282
1283   class AdvSIMD_SVE_TBL_Intrinsic
1284     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1285                 [LLVMMatchType<0>,
1286                  LLVMVectorOfBitcastsToInt<0>],
1287                 [IntrNoMem]>;
1288
1289   class AdvSIMD_SVE2_TBX_Intrinsic
1290     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1291                 [LLVMMatchType<0>,
1292                  LLVMMatchType<0>,
1293                  LLVMVectorOfBitcastsToInt<0>],
1294                 [IntrNoMem]>;
1295
1296   class SVE2_1VectorArg_Long_Intrinsic
1297     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1298                 [LLVMSubdivide2VectorType<0>,
1299                  llvm_i32_ty],
1300                 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1301
1302   class SVE2_2VectorArg_Long_Intrinsic
1303     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1304                 [LLVMSubdivide2VectorType<0>,
1305                  LLVMSubdivide2VectorType<0>],
1306                 [IntrNoMem]>;
1307
1308   class SVE2_2VectorArgIndexed_Long_Intrinsic
1309   : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1310               [LLVMSubdivide2VectorType<0>,
1311                LLVMSubdivide2VectorType<0>,
1312                llvm_i32_ty],
1313               [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1314
1315   class SVE2_2VectorArg_Wide_Intrinsic
1316     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1317                 [LLVMMatchType<0>,
1318                  LLVMSubdivide2VectorType<0>],
1319                 [IntrNoMem]>;
1320
1321   class SVE2_2VectorArg_Pred_Long_Intrinsic
1322     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1323                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1324                  LLVMMatchType<0>,
1325                  LLVMSubdivide2VectorType<0>],
1326                 [IntrNoMem]>;
1327
1328   class SVE2_3VectorArg_Long_Intrinsic
1329     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1330                 [LLVMMatchType<0>,
1331                  LLVMSubdivide2VectorType<0>,
1332                  LLVMSubdivide2VectorType<0>],
1333                 [IntrNoMem]>;
1334
1335   class SVE2_3VectorArgIndexed_Long_Intrinsic
1336     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1337                 [LLVMMatchType<0>,
1338                  LLVMSubdivide2VectorType<0>,
1339                  LLVMSubdivide2VectorType<0>,
1340                  llvm_i32_ty],
1341                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1342
1343   class SVE2_1VectorArg_Narrowing_Intrinsic
1344     : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1345                 [llvm_anyvector_ty],
1346                 [IntrNoMem]>;
1347
1348   class SVE2_Merged1VectorArg_Narrowing_Intrinsic
1349     : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1350                 [LLVMSubdivide2VectorType<0>,
1351                  llvm_anyvector_ty],
1352                 [IntrNoMem]>;
1353   class SVE2_2VectorArg_Narrowing_Intrinsic
1354       : DefaultAttrsIntrinsic<
1355             [LLVMSubdivide2VectorType<0>],
1356             [llvm_anyvector_ty, LLVMMatchType<0>],
1357             [IntrNoMem]>;
1358
1359   class SVE2_Merged2VectorArg_Narrowing_Intrinsic
1360       : DefaultAttrsIntrinsic<
1361             [LLVMSubdivide2VectorType<0>],
1362             [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
1363             [IntrNoMem]>;
1364
1365   class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
1366       : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1367                   [llvm_anyvector_ty, llvm_i32_ty],
1368                   [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1369
1370   class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
1371       : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1372                   [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
1373                    llvm_i32_ty],
1374                   [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1375
1376   class SVE2_CONFLICT_DETECT_Intrinsic
1377     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1378                 [LLVMAnyPointerType<llvm_any_ty>,
1379                  LLVMMatchType<1>]>;
1380
1381   class SVE2_3VectorArg_Indexed_Intrinsic
1382     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1383                 [LLVMMatchType<0>,
1384                  LLVMSubdivide2VectorType<0>,
1385                  LLVMSubdivide2VectorType<0>,
1386                  llvm_i32_ty],
1387                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1388
1389   class AdvSIMD_SVE_CDOT_LANE_Intrinsic
1390     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1391                 [LLVMMatchType<0>,
1392                  LLVMSubdivide4VectorType<0>,
1393                  LLVMSubdivide4VectorType<0>,
1394                  llvm_i32_ty,
1395                  llvm_i32_ty],
1396                 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1397
1398   // NOTE: There is no relationship between these intrinsics beyond an attempt
1399   // to reuse currently identical class definitions.
1400   class AdvSIMD_SVE_LOGB_Intrinsic  : AdvSIMD_SVE_CNT_Intrinsic;
1401   class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
1402   class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;
1403
1404   // This class of intrinsics are not intended to be useful within LLVM IR but
1405   // are instead here to support some of the more regid parts of the ACLE.
1406   class Builtin_SVCVT<LLVMType OUT, LLVMType PRED, LLVMType IN>
1407       : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
1408 }
1409
1410 //===----------------------------------------------------------------------===//
1411 // SVE
1412
1413 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
1414
1415 class AdvSIMD_SVE_Reduce_Intrinsic
1416   : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
1417               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1418                llvm_anyvector_ty],
1419               [IntrNoMem]>;
1420
1421 class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
1422   : DefaultAttrsIntrinsic<[llvm_i64_ty],
1423               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1424                llvm_anyvector_ty],
1425               [IntrNoMem]>;
1426
1427 class AdvSIMD_SVE_WHILE_Intrinsic
1428     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1429                 [llvm_anyint_ty, LLVMMatchType<1>],
1430                 [IntrNoMem]>;
1431
1432 class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
1433     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1434                 [
1435                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1436                   LLVMPointerToElt<0>,
1437                   LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1438                 ],
1439                 [IntrReadMem, IntrArgMemOnly]>;
1440
1441 class AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic
1442     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1443                 [
1444                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1445                   LLVMPointerToElt<0>,
1446                   LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1447                 ],
1448                 [IntrInaccessibleMemOrArgMemOnly]>;
1449
1450 class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
1451     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1452                 [
1453                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1454                   LLVMPointerToElt<0>,
1455                   LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1456                 ],
1457                 [IntrReadMem, IntrArgMemOnly]>;
1458
1459 class AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic
1460     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1461                 [
1462                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1463                   LLVMPointerToElt<0>,
1464                   LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1465                 ],
1466                 [IntrInaccessibleMemOrArgMemOnly]>;
1467
1468 class AdvSIMD_GatherLoad_VS_Intrinsic
1469     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1470                 [
1471                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1472                   llvm_anyvector_ty,
1473                   llvm_i64_ty
1474                 ],
1475                 [IntrReadMem]>;
1476
1477 class AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic
1478     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1479                 [
1480                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1481                   llvm_anyvector_ty,
1482                   llvm_i64_ty
1483                 ],
1484                 [IntrInaccessibleMemOrArgMemOnly]>;
1485
1486 class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
1487     : DefaultAttrsIntrinsic<[],
1488                [
1489                  llvm_anyvector_ty,
1490                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1491                  LLVMPointerToElt<0>,
1492                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1493                ],
1494                [IntrWriteMem, IntrArgMemOnly]>;
1495
1496 class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
1497     : DefaultAttrsIntrinsic<[],
1498                [
1499                  llvm_anyvector_ty,
1500                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1501                  LLVMPointerToElt<0>,
1502                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1503                ],
1504                [IntrWriteMem, IntrArgMemOnly]>;
1505
1506 class AdvSIMD_ScatterStore_VS_Intrinsic
1507     : DefaultAttrsIntrinsic<[],
1508                [
1509                  llvm_anyvector_ty,
1510                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1511                  llvm_anyvector_ty, llvm_i64_ty
1512                ],
1513                [IntrWriteMem]>;
1514
1515
1516 class SVE_gather_prf_SV
1517     : DefaultAttrsIntrinsic<[],
1518                 [
1519                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1520                   llvm_ptr_ty, // Base address
1521                   llvm_anyvector_ty, // Offsets
1522                   llvm_i32_ty // Prfop
1523                 ],
1524                 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;
1525
1526 class SVE_gather_prf_VS
1527     : DefaultAttrsIntrinsic<[],
1528                 [
1529                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1530                   llvm_anyvector_ty, // Base addresses
1531                   llvm_i64_ty, // Scalar offset
1532                   llvm_i32_ty // Prfop
1533                 ],
1534                 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;
1535
1536 class SVE_MatMul_Intrinsic
1537     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1538                 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
1539                 [IntrNoMem]>;
1540
1541 class SVE_4Vec_BF16
1542     : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
1543                 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
1544                 [IntrNoMem]>;
1545
1546 class SVE_4Vec_BF16_Indexed
1547     : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
1548                 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
1549                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1550
1551 //
1552 // Vector tuple creation intrinsics (ACLE)
1553 //
1554
1555 def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
1556 def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
1557 def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;
1558
1559 //
1560 // Vector tuple insertion/extraction intrinsics (ACLE)
1561 //
1562
1563 def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
1564 def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;
1565
1566 //
1567 // Loads
1568 //
1569
1570 def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;
1571
1572 def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1573 def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1574 def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1575
1576 def int_aarch64_sve_ld2_sret : AdvSIMD_2Vec_PredLoad_Intrinsic;
1577 def int_aarch64_sve_ld3_sret : AdvSIMD_3Vec_PredLoad_Intrinsic;
1578 def int_aarch64_sve_ld4_sret : AdvSIMD_4Vec_PredLoad_Intrinsic;
1579
1580 def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1581 def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic;
1582 def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic;
1583
1584 def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1585 def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
1586
1587 //
1588 // Stores
1589 //
1590
1591 def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
1592 def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
1593 def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
1594 def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;
1595
1596 def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1597
1598 //
1599 // Prefetches
1600 //
1601
1602 def int_aarch64_sve_prf
1603   : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
1604                   [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
1605
1606 // Scalar + 32-bit scaled offset vector, zero extend, packed and
1607 // unpacked.
1608 def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
1609 def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
1610 def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
1611 def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;
1612
1613 // Scalar + 32-bit scaled offset vector, sign extend, packed and
1614 // unpacked.
1615 def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
1616 def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
1617 def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
1618 def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;
1619
1620 // Scalar + 64-bit scaled offset vector.
1621 def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
1622 def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
1623 def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
1624 def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;
1625
1626 // Vector + scalar.
1627 def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
1628 def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
1629 def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
1630 def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;
1631
1632 //
1633 // Scalar to vector operations
1634 //
1635
1636 def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
1637 def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;
1638
1639 def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;
1640
1641 //
1642 // Address calculation
1643 //
1644
1645 def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
1646 def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
1647 def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
1648 def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;
1649
1650 //
1651 // Integer arithmetic
1652 //
1653
1654 def int_aarch64_sve_add   : AdvSIMD_Pred2VectorArg_Intrinsic;
1655 def int_aarch64_sve_sub   : AdvSIMD_Pred2VectorArg_Intrinsic;
1656 def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1657
1658 def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;
1659
1660 def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
1661 def int_aarch64_sve_mul_lane   : AdvSIMD_2VectorArgIndexed_Intrinsic;
1662 def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1663 def int_aarch64_sve_umulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1664
1665 def int_aarch64_sve_sdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1666 def int_aarch64_sve_udiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1667 def int_aarch64_sve_sdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1668 def int_aarch64_sve_udivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1669
1670 def int_aarch64_sve_smax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1671 def int_aarch64_sve_umax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1672 def int_aarch64_sve_smin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1673 def int_aarch64_sve_umin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1674 def int_aarch64_sve_sabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1675 def int_aarch64_sve_uabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1676
1677 def int_aarch64_sve_mad        : AdvSIMD_Pred3VectorArg_Intrinsic;
1678 def int_aarch64_sve_msb        : AdvSIMD_Pred3VectorArg_Intrinsic;
1679 def int_aarch64_sve_mla        : AdvSIMD_Pred3VectorArg_Intrinsic;
1680 def int_aarch64_sve_mla_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1681 def int_aarch64_sve_mls        : AdvSIMD_Pred3VectorArg_Intrinsic;
1682 def int_aarch64_sve_mls_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1683
1684 def int_aarch64_sve_saddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1685 def int_aarch64_sve_uaddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1686
1687 def int_aarch64_sve_smaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1688 def int_aarch64_sve_umaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1689 def int_aarch64_sve_sminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1690 def int_aarch64_sve_uminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1691
1692 def int_aarch64_sve_orv        : AdvSIMD_SVE_Reduce_Intrinsic;
1693 def int_aarch64_sve_eorv       : AdvSIMD_SVE_Reduce_Intrinsic;
1694 def int_aarch64_sve_andv       : AdvSIMD_SVE_Reduce_Intrinsic;
1695
1696 def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
1697 def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
1698
1699 def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
1700 def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1701
1702 def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
1703 def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1704
1705 def int_aarch64_sve_sqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1706 def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1707 def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1708 def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1709
1710 // Shifts
1711
1712 def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1713 def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1714 def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
1715 def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
1716 def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
1717 def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1718 def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1719 def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1720
1721 //
1722 // Integer comparisons
1723 //
1724
1725 def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1726 def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
1727 def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1728 def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
1729 def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
1730 def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
1731
1732 def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1733 def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1734 def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1735 def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1736 def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1737 def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1738 def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1739 def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1740 def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1741 def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1742
1743 //
1744 // Counting bits
1745 //
1746
1747 def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
1748 def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
1749 def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
1750
1751 //
1752 // Counting elements
1753 //
1754
1755 def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
1756 def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
1757 def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
1758 def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
1759
1760 def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
1761
1762 //
1763 // FFR manipulation
1764 //
1765
1766 def int_aarch64_sve_rdffr   : ClangBuiltin<"__builtin_sve_svrdffr">,   DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>;
1767 def int_aarch64_sve_rdffr_z : ClangBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>;
1768 def int_aarch64_sve_setffr  : ClangBuiltin<"__builtin_sve_svsetffr">,  DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>;
1769 def int_aarch64_sve_wrffr   : ClangBuiltin<"__builtin_sve_svwrffr">,   DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>;
1770
1771 //
1772 // Saturating scalar arithmetic
1773 //
1774
1775 def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1776 def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1777 def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1778 def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1779
1780 def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1781 def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1782 def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1783 def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1784 def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1785 def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1786 def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1787 def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1788 def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1789 def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1790
1791 def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1792 def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1793 def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1794 def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1795
1796 def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1797 def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1798 def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1799 def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1800 def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1801 def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1802 def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1803 def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1804 def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1805 def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1806
1807 def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1808 def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1809 def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1810 def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1811
1812 def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1813 def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1814 def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1815 def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1816 def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1817 def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1818 def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1819 def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1820 def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1821 def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1822
1823 def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1824 def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1825 def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1826 def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1827
1828 def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1829 def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1830 def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1831 def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1832 def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1833 def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1834 def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1835 def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1836 def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1837 def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1838
1839 //
1840 // Reversal
1841 //
1842
1843 def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
1844 def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
1845 def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
1846 def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;
1847
1848 //
1849 // Permutations and selection
1850 //
1851
1852 def int_aarch64_sve_clasta    : AdvSIMD_Pred2VectorArg_Intrinsic;
1853 def int_aarch64_sve_clasta_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1854 def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
1855 def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1856 def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
1857 def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
1858 def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
1859 def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
1860 def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
1861 def int_aarch64_sve_lastb     : AdvSIMD_SVE_Reduce_Intrinsic;
1862 def int_aarch64_sve_rev       : AdvSIMD_1VectorArg_Intrinsic;
1863 def int_aarch64_sve_splice    : AdvSIMD_Pred2VectorArg_Intrinsic;
1864 def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1865 def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1866 def int_aarch64_sve_tbl       : AdvSIMD_SVE_TBL_Intrinsic;
1867 def int_aarch64_sve_trn1      : AdvSIMD_2VectorArg_Intrinsic;
1868 def int_aarch64_sve_trn2      : AdvSIMD_2VectorArg_Intrinsic;
1869 def int_aarch64_sve_trn1q     : AdvSIMD_2VectorArg_Intrinsic;
1870 def int_aarch64_sve_trn2q     : AdvSIMD_2VectorArg_Intrinsic;
1871 def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1872 def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1873 def int_aarch64_sve_uzp1      : AdvSIMD_2VectorArg_Intrinsic;
1874 def int_aarch64_sve_uzp2      : AdvSIMD_2VectorArg_Intrinsic;
1875 def int_aarch64_sve_uzp1q     : AdvSIMD_2VectorArg_Intrinsic;
1876 def int_aarch64_sve_uzp2q     : AdvSIMD_2VectorArg_Intrinsic;
1877 def int_aarch64_sve_zip1      : AdvSIMD_2VectorArg_Intrinsic;
1878 def int_aarch64_sve_zip2      : AdvSIMD_2VectorArg_Intrinsic;
1879 def int_aarch64_sve_zip1q     : AdvSIMD_2VectorArg_Intrinsic;
1880 def int_aarch64_sve_zip2q     : AdvSIMD_2VectorArg_Intrinsic;
1881
1882 //
1883 // Logical operations
1884 //
1885
1886 def int_aarch64_sve_and  : AdvSIMD_Pred2VectorArg_Intrinsic;
1887 def int_aarch64_sve_bic  : AdvSIMD_Pred2VectorArg_Intrinsic;
1888 def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
1889 def int_aarch64_sve_eor  : AdvSIMD_Pred2VectorArg_Intrinsic;
1890 def int_aarch64_sve_not  : AdvSIMD_Merged1VectorArg_Intrinsic;
1891 def int_aarch64_sve_orr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1892
1893 //
1894 // Conversion
1895 //
1896
1897 def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1898 def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1899 def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1900 def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1901 def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1902 def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1903
1904 //
1905 // While comparisons
1906 //
1907
1908 def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
1909 def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
1910 def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
1911 def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
1912 def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
1913 def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
1914 def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
1915 def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;
1916
1917 //
1918 // Floating-point arithmetic
1919 //
1920
1921 def int_aarch64_sve_fabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1922 def int_aarch64_sve_fabs       : AdvSIMD_Merged1VectorArg_Intrinsic;
1923 def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1924 def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
1925 def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
1926 def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
1927 def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1928 def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1929 def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
1930 def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
1931 def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1932 def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1933 def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1934 def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1935 def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
1936 def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1937 def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
1938 def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1939 def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;
1940 def int_aarch64_sve_fmul       : AdvSIMD_Pred2VectorArg_Intrinsic;
1941 def int_aarch64_sve_fmulx      : AdvSIMD_Pred2VectorArg_Intrinsic;
1942 def int_aarch64_sve_fneg       : AdvSIMD_Merged1VectorArg_Intrinsic;
1943 def int_aarch64_sve_fmul_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
1944 def int_aarch64_sve_fnmad      : AdvSIMD_Pred3VectorArg_Intrinsic;
1945 def int_aarch64_sve_fnmla      : AdvSIMD_Pred3VectorArg_Intrinsic;
1946 def int_aarch64_sve_fnmls      : AdvSIMD_Pred3VectorArg_Intrinsic;
1947 def int_aarch64_sve_fnmsb      : AdvSIMD_Pred3VectorArg_Intrinsic;
1948 def int_aarch64_sve_frecpe_x   : AdvSIMD_1VectorArg_Intrinsic;
1949 def int_aarch64_sve_frecps_x   : AdvSIMD_2VectorArg_Intrinsic;
1950 def int_aarch64_sve_frecpx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1951 def int_aarch64_sve_frinta     : AdvSIMD_Merged1VectorArg_Intrinsic;
1952 def int_aarch64_sve_frinti     : AdvSIMD_Merged1VectorArg_Intrinsic;
1953 def int_aarch64_sve_frintm     : AdvSIMD_Merged1VectorArg_Intrinsic;
1954 def int_aarch64_sve_frintn     : AdvSIMD_Merged1VectorArg_Intrinsic;
1955 def int_aarch64_sve_frintp     : AdvSIMD_Merged1VectorArg_Intrinsic;
1956 def int_aarch64_sve_frintx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1957 def int_aarch64_sve_frintz     : AdvSIMD_Merged1VectorArg_Intrinsic;
1958 def int_aarch64_sve_frsqrte_x  : AdvSIMD_1VectorArg_Intrinsic;
1959 def int_aarch64_sve_frsqrts_x  : AdvSIMD_2VectorArg_Intrinsic;
1960 def int_aarch64_sve_fscale     : AdvSIMD_SVE_SCALE_Intrinsic;
1961 def int_aarch64_sve_fsqrt      : AdvSIMD_Merged1VectorArg_Intrinsic;
1962 def int_aarch64_sve_fsub       : AdvSIMD_Pred2VectorArg_Intrinsic;
1963 def int_aarch64_sve_fsubr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1964 def int_aarch64_sve_ftmad_x    : AdvSIMD_2VectorArgIndexed_Intrinsic;
1965 def int_aarch64_sve_ftsmul_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1966 def int_aarch64_sve_ftssel_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1967
1968 //
1969 // Floating-point reductions
1970 //
1971
1972 def int_aarch64_sve_fadda   : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1973 def int_aarch64_sve_faddv   : AdvSIMD_SVE_Reduce_Intrinsic;
1974 def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
1975 def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1976 def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
1977 def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1978
1979 //
1980 // Floating-point conversions
1981 //
1982
1983 def int_aarch64_sve_fcvt   : AdvSIMD_SVE_FCVT_Intrinsic;
1984 def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
1985 def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
1986 def int_aarch64_sve_scvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1987 def int_aarch64_sve_ucvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1988
1989 //
1990 // Floating-point comparisons
1991 //
1992
1993 def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
1994 def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;
1995
1996 def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1997 def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
1998 def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1999 def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
2000 def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;
2001
2002 def int_aarch64_sve_fcvtzs_i32f16   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
2003 def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2004 def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
2005 def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
2006
2007 def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
2008 def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
2009
2010 def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
2011 def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2012 def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
2013 def int_aarch64_sve_fcvtzu_i64f32   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
2014
2015 def int_aarch64_sve_fcvt_f16f32     : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
2016 def int_aarch64_sve_fcvt_f16f64     : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2017 def int_aarch64_sve_fcvt_f32f64     : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2018
2019 def int_aarch64_sve_fcvt_f32f16     : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
2020 def int_aarch64_sve_fcvt_f64f16     : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
2021 def int_aarch64_sve_fcvt_f64f32     : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
2022
2023 def int_aarch64_sve_fcvtlt_f32f16   : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
2024 def int_aarch64_sve_fcvtlt_f64f32   : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
2025 def int_aarch64_sve_fcvtnt_f16f32   : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
2026 def int_aarch64_sve_fcvtnt_f32f64   : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2027
2028 def int_aarch64_sve_fcvtx_f32f64    : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2029 def int_aarch64_sve_fcvtxnt_f32f64  : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
2030
2031 def int_aarch64_sve_scvtf_f16i32    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
2032 def int_aarch64_sve_scvtf_f16i64    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
2033 def int_aarch64_sve_scvtf_f32i64    : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
2034 def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
2035
2036 def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
2037 def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
2038 def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
2039 def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
2040
2041 //
2042 // Predicate creation
2043 //
2044
2045 def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
2046
2047 //
2048 // Predicate operations
2049 //
2050
2051 def int_aarch64_sve_and_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
2052 def int_aarch64_sve_bic_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
2053 def int_aarch64_sve_brka    : AdvSIMD_Merged1VectorArg_Intrinsic;
2054 def int_aarch64_sve_brka_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
2055 def int_aarch64_sve_brkb    : AdvSIMD_Merged1VectorArg_Intrinsic;
2056 def int_aarch64_sve_brkb_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
2057 def int_aarch64_sve_brkn_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
2058 def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
2059 def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
2060 def int_aarch64_sve_eor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
2061 def int_aarch64_sve_nand_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
2062 def int_aarch64_sve_nor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
2063 def int_aarch64_sve_orn_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
2064 def int_aarch64_sve_orr_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
2065 def int_aarch64_sve_pfirst  : AdvSIMD_Pred1VectorArg_Intrinsic;
2066 def int_aarch64_sve_pnext   : AdvSIMD_Pred1VectorArg_Intrinsic;
2067 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
2068 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
2069
2070 //
2071 // Testing predicates
2072 //
2073
2074 def int_aarch64_sve_ptest_any   : AdvSIMD_SVE_PTEST_Intrinsic;
2075 def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
2076 def int_aarch64_sve_ptest_last  : AdvSIMD_SVE_PTEST_Intrinsic;
2077
2078 //
2079 // Reinterpreting data
2080 //
2081
2082 def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2083                                                     [llvm_nxv16i1_ty],
2084                                                     [IntrNoMem]>;
2085
2086 def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty],
2087                                                   [llvm_anyvector_ty],
2088                                                   [IntrNoMem]>;
2089
2090 //
2091 // Gather loads: scalar base + vector offsets
2092 //
2093
2094 // 64 bit unscaled offsets
2095 def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
2096
2097 // 64 bit scaled offsets
2098 def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
2099
2100 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2101 def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
2102 def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
2103
2104 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2105 def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
2106 def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
2107
2108 //
2109 // Gather loads: vector base + scalar offset
2110 //
2111
2112 def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
2113
2114
2115 //
2116 // First-faulting gather loads: scalar base + vector offsets
2117 //
2118
2119 // 64 bit unscaled offsets
2120 def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic;
2121
2122 // 64 bit scaled offsets
2123 def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic;
2124
2125 // 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
2126 def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;
2127 def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;
2128
2129 // 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
2130 def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;
2131 def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;
2132
2133 //
2134 // First-faulting gather loads: vector base + scalar offset
2135 //
2136
2137 def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic;
2138
2139
2140 //
2141 // Non-temporal gather loads: scalar base + vector offsets
2142 //
2143
2144 // 64 bit unscaled offsets
2145 def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
2146
2147 // 64 bit indices
2148 def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
2149
2150 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2151 def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
2152
2153 //
2154 // Non-temporal gather loads: vector base + scalar offset
2155 //
2156
2157 def int_aarch64_sve_ldnt1_gather_scalar_offset  : AdvSIMD_GatherLoad_VS_Intrinsic;
2158
2159 //
2160 // Scatter stores: scalar base + vector offsets
2161 //
2162
2163 // 64 bit unscaled offsets
2164 def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2165
2166 // 64 bit scaled offsets
2167 def int_aarch64_sve_st1_scatter_index
2168     : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2169
2170 // 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2171 def int_aarch64_sve_st1_scatter_sxtw
2172     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2173
2174 def int_aarch64_sve_st1_scatter_uxtw
2175     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2176
2177 // 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2178 def int_aarch64_sve_st1_scatter_sxtw_index
2179     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2180
2181 def int_aarch64_sve_st1_scatter_uxtw_index
2182     : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2183
2184 //
2185 // Scatter stores: vector base + scalar offset
2186 //
2187
2188 def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
2189
2190 //
2191 // Non-temporal scatter stores: scalar base + vector offsets
2192 //
2193
2194 // 64 bit unscaled offsets
2195 def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2196
2197 // 64 bit indices
2198 def int_aarch64_sve_stnt1_scatter_index
2199     : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2200
2201 // 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2202 def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2203
2204 //
2205 // Non-temporal scatter stores: vector base + scalar offset
2206 //
2207
2208 def int_aarch64_sve_stnt1_scatter_scalar_offset  : AdvSIMD_ScatterStore_VS_Intrinsic;
2209
2210 //
2211 // SVE2 - Uniform DSP operations
2212 //
2213
2214 def int_aarch64_sve_saba          : AdvSIMD_3VectorArg_Intrinsic;
2215 def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2216 def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2217 def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2218 def int_aarch64_sve_sli           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2219 def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
2220 def int_aarch64_sve_sqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2221 def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
2222 def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
2223 def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
2224 def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
2225 def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2226 def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
2227 def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2228 def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
2229 def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2230 def int_aarch64_sve_sqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2231 def int_aarch64_sve_sqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2232 def int_aarch64_sve_sqshlu        : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2233 def int_aarch64_sve_sqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2234 def int_aarch64_sve_sqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2235 def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2236 def int_aarch64_sve_sri           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2237 def int_aarch64_sve_srshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2238 def int_aarch64_sve_srshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2239 def int_aarch64_sve_srsra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2240 def int_aarch64_sve_ssra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2241 def int_aarch64_sve_suqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2242 def int_aarch64_sve_uaba          : AdvSIMD_3VectorArg_Intrinsic;
2243 def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2244 def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2245 def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2246 def int_aarch64_sve_uqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2247 def int_aarch64_sve_uqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2248 def int_aarch64_sve_uqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2249 def int_aarch64_sve_uqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2250 def int_aarch64_sve_uqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2251 def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
2252 def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2253 def int_aarch64_sve_urshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2254 def int_aarch64_sve_urshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2255 def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;
2256 def int_aarch64_sve_ursra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2257 def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2258 def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2259
2260 //
2261 // SVE2 - Widening DSP operations
2262 //
2263
2264 def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
2265 def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
2266 def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
2267 def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
2268 def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
2269 def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
2270 def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
2271 def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
2272 def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
2273 def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
2274 def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
2275 def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
2276 def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
2277 def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
2278 def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
2279 def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
2280 def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
2281 def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
2282 def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
2283 def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
2284 def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
2285 def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
2286 def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
2287 def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
2288 def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
2289 def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
2290 def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
2291 def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;
2292
2293 //
2294 // SVE2 - Non-widening pairwise arithmetic
2295 //
2296
2297 def int_aarch64_sve_addp    : AdvSIMD_Pred2VectorArg_Intrinsic;
2298 def int_aarch64_sve_faddp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2299 def int_aarch64_sve_fmaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2300 def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2301 def int_aarch64_sve_fminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2302 def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2303 def int_aarch64_sve_smaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2304 def int_aarch64_sve_sminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2305 def int_aarch64_sve_umaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2306 def int_aarch64_sve_uminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2307
2308 //
2309 // SVE2 - Widening pairwise arithmetic
2310 //
2311
2312 def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2313 def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2314
2315 //
2316 // SVE2 - Uniform complex integer arithmetic
2317 //
2318
2319 def int_aarch64_sve_cadd_x           : AdvSIMD_SVE2_CADD_Intrinsic;
2320 def int_aarch64_sve_sqcadd_x         : AdvSIMD_SVE2_CADD_Intrinsic;
2321 def int_aarch64_sve_cmla_x           : AdvSIMD_SVE2_CMLA_Intrinsic;
2322 def int_aarch64_sve_cmla_lane_x      : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2323 def int_aarch64_sve_sqrdcmlah_x      : AdvSIMD_SVE2_CMLA_Intrinsic;
2324 def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2325
2326 //
2327 // SVE2 - Widening complex integer arithmetic
2328 //
2329
2330 def int_aarch64_sve_saddlbt   : SVE2_2VectorArg_Long_Intrinsic;
2331 def int_aarch64_sve_ssublbt   : SVE2_2VectorArg_Long_Intrinsic;
2332 def int_aarch64_sve_ssubltb   : SVE2_2VectorArg_Long_Intrinsic;
2333
2334 //
2335 // SVE2 - Widening complex integer dot product
2336 //
2337
2338 def int_aarch64_sve_cdot      : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2339 def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;
2340
2341 //
2342 // SVE2 - Floating-point widening multiply-accumulate
2343 //
2344
2345 def int_aarch64_sve_fmlalb        : SVE2_3VectorArg_Long_Intrinsic;
2346 def int_aarch64_sve_fmlalb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2347 def int_aarch64_sve_fmlalt        : SVE2_3VectorArg_Long_Intrinsic;
2348 def int_aarch64_sve_fmlalt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2349 def int_aarch64_sve_fmlslb        : SVE2_3VectorArg_Long_Intrinsic;
2350 def int_aarch64_sve_fmlslb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2351 def int_aarch64_sve_fmlslt        : SVE2_3VectorArg_Long_Intrinsic;
2352 def int_aarch64_sve_fmlslt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2353
2354 //
2355 // SVE2 - Floating-point integer binary logarithm
2356 //
2357
2358 def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;
2359
2360 //
2361 // SVE2 - Vector histogram count
2362 //
2363
2364 def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
2365 def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;
2366
2367 //
2368 // SVE2 - Character match
2369 //
2370
2371 def int_aarch64_sve_match   : AdvSIMD_SVE_Compare_Intrinsic;
2372 def int_aarch64_sve_nmatch  : AdvSIMD_SVE_Compare_Intrinsic;
2373
2374 //
2375 // SVE2 - Unary narrowing operations
2376 //
2377
2378 def int_aarch64_sve_sqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2379 def int_aarch64_sve_sqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2380 def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
2381 def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2382 def int_aarch64_sve_uqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2383 def int_aarch64_sve_uqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2384
2385 //
2386 // SVE2 - Binary narrowing DSP operations
2387 //
2388 def int_aarch64_sve_addhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2389 def int_aarch64_sve_addhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2390
2391 def int_aarch64_sve_raddhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2392 def int_aarch64_sve_raddhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2393
2394 def int_aarch64_sve_subhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2395 def int_aarch64_sve_subhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2396
2397 def int_aarch64_sve_rsubhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2398 def int_aarch64_sve_rsubhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2399
2400 // Narrowing shift right
2401 def int_aarch64_sve_shrnb     : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2402 def int_aarch64_sve_shrnt     : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2403
2404 def int_aarch64_sve_rshrnb    : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2405 def int_aarch64_sve_rshrnt    : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2406
2407 // Saturating shift right - signed input/output
2408 def int_aarch64_sve_sqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2409 def int_aarch64_sve_sqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2410
2411 def int_aarch64_sve_sqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2412 def int_aarch64_sve_sqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2413
2414 // Saturating shift right - unsigned input/output
2415 def int_aarch64_sve_uqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2416 def int_aarch64_sve_uqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2417
2418 def int_aarch64_sve_uqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2419 def int_aarch64_sve_uqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2420
2421 // Saturating shift right - signed input, unsigned output
2422 def int_aarch64_sve_sqshrunb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2423 def int_aarch64_sve_sqshrunt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2424
2425 def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2426 def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2427
2428 // SVE2 MLA LANE.
2429 def int_aarch64_sve_smlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2430 def int_aarch64_sve_smlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2431 def int_aarch64_sve_umlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2432 def int_aarch64_sve_umlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2433 def int_aarch64_sve_smlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2434 def int_aarch64_sve_smlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2435 def int_aarch64_sve_umlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2436 def int_aarch64_sve_umlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2437 def int_aarch64_sve_smullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2438 def int_aarch64_sve_smullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2439 def int_aarch64_sve_umullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2440 def int_aarch64_sve_umullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2441 def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2442 def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2443 def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2444 def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2445 def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2446 def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2447
2448 // SVE2 MLA Unpredicated.
2449 def int_aarch64_sve_smlalb      : SVE2_3VectorArg_Long_Intrinsic;
2450 def int_aarch64_sve_smlalt      : SVE2_3VectorArg_Long_Intrinsic;
2451 def int_aarch64_sve_umlalb      : SVE2_3VectorArg_Long_Intrinsic;
2452 def int_aarch64_sve_umlalt      : SVE2_3VectorArg_Long_Intrinsic;
2453 def int_aarch64_sve_smlslb      : SVE2_3VectorArg_Long_Intrinsic;
2454 def int_aarch64_sve_smlslt      : SVE2_3VectorArg_Long_Intrinsic;
2455 def int_aarch64_sve_umlslb      : SVE2_3VectorArg_Long_Intrinsic;
2456 def int_aarch64_sve_umlslt      : SVE2_3VectorArg_Long_Intrinsic;
2457 def int_aarch64_sve_smullb      : SVE2_2VectorArg_Long_Intrinsic;
2458 def int_aarch64_sve_smullt      : SVE2_2VectorArg_Long_Intrinsic;
2459 def int_aarch64_sve_umullb      : SVE2_2VectorArg_Long_Intrinsic;
2460 def int_aarch64_sve_umullt      : SVE2_2VectorArg_Long_Intrinsic;
2461
2462 def int_aarch64_sve_sqdmlalb    : SVE2_3VectorArg_Long_Intrinsic;
2463 def int_aarch64_sve_sqdmlalt    : SVE2_3VectorArg_Long_Intrinsic;
2464 def int_aarch64_sve_sqdmlslb    : SVE2_3VectorArg_Long_Intrinsic;
2465 def int_aarch64_sve_sqdmlslt    : SVE2_3VectorArg_Long_Intrinsic;
2466 def int_aarch64_sve_sqdmullb    : SVE2_2VectorArg_Long_Intrinsic;
2467 def int_aarch64_sve_sqdmullt    : SVE2_2VectorArg_Long_Intrinsic;
2468 def int_aarch64_sve_sqdmlalbt   : SVE2_3VectorArg_Long_Intrinsic;
2469 def int_aarch64_sve_sqdmlslbt   : SVE2_3VectorArg_Long_Intrinsic;
2470
2471 // SVE2 ADDSUB Long Unpredicated.
2472 def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
2473 def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
2474 def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
2475 def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;
2476
2477 //
2478 // SVE2 - Polynomial arithmetic
2479 //
2480 def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
2481 def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
2482 def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
2483 def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
2484
2485 //
2486 // SVE2 bitwise ternary operations.
2487 //
2488 def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
2489 def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
2490 def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
2491 def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
2492 def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
2493 def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
2494 def int_aarch64_sve_xar    : AdvSIMD_2VectorArgIndexed_Intrinsic;
2495
2496 //
2497 // SVE2 - Optional AES, SHA-3 and SM4
2498 //
2499
2500 def int_aarch64_sve_aesd    : ClangBuiltin<"__builtin_sve_svaesd_u8">,
2501                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2502                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2503                                         [IntrNoMem]>;
2504 def int_aarch64_sve_aesimc  : ClangBuiltin<"__builtin_sve_svaesimc_u8">,
2505                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2506                                         [llvm_nxv16i8_ty],
2507                                         [IntrNoMem]>;
2508 def int_aarch64_sve_aese    : ClangBuiltin<"__builtin_sve_svaese_u8">,
2509                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2510                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2511                                         [IntrNoMem]>;
2512 def int_aarch64_sve_aesmc   : ClangBuiltin<"__builtin_sve_svaesmc_u8">,
2513                               DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2514                                         [llvm_nxv16i8_ty],
2515                                         [IntrNoMem]>;
2516 def int_aarch64_sve_rax1    : ClangBuiltin<"__builtin_sve_svrax1_u64">,
2517                               DefaultAttrsIntrinsic<[llvm_nxv2i64_ty],
2518                                         [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
2519                                         [IntrNoMem]>;
2520 def int_aarch64_sve_sm4e    : ClangBuiltin<"__builtin_sve_svsm4e_u32">,
2521                               DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
2522                                         [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2523                                         [IntrNoMem]>;
2524 def int_aarch64_sve_sm4ekey : ClangBuiltin<"__builtin_sve_svsm4ekey_u32">,
2525                               DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
2526                                         [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2527                                         [IntrNoMem]>;
2528 //
2529 // SVE2 - Extended table lookup/permute
2530 //
2531
2532 def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
2533 def int_aarch64_sve_tbx  : AdvSIMD_SVE2_TBX_Intrinsic;
2534
2535 //
2536 // SVE2 - Optional bit permutation
2537 //
2538
2539 def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
2540 def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
2541 def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;
2542
2543
2544 //
2545 // SVE ACLE: 7.3. INT8 matrix multiply extensions
2546 //
2547 def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
2548 def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
2549 def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;
2550
2551 def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
2552 def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2553 def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2554
2555 //
2556 // SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
2557 //
2558 def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
2559
2560 //
2561 // SVE ACLE: 7.2. BFloat16 extensions
2562 //
2563
2564 def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
2565 def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
2566 def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
2567
2568 def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;
2569
2570 def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
2571 def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
2572 def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
2573 }
2574
2575 //
2576 // SVE2 - Contiguous conflict detection
2577 //
2578
2579 def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2580 def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2581 def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2582 def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2583 def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2584 def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2585 def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2586 def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;
2587
2588 // Scalable Matrix Extension (SME) Intrinsics
2589 let TargetPrefix = "aarch64" in {
2590   class SME_Load_Store_Intrinsic<LLVMType pred_ty>
2591     : DefaultAttrsIntrinsic<[],
2592         [pred_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], []>;
2593
2594   // Loads
2595   def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
2596   def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
2597   def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
2598   def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
2599   def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
2600   def int_aarch64_sme_ld1b_vert  : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
2601   def int_aarch64_sme_ld1h_vert  : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
2602   def int_aarch64_sme_ld1w_vert  : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
2603   def int_aarch64_sme_ld1d_vert  : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
2604   def int_aarch64_sme_ld1q_vert  : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
2605
2606   // Stores
2607   def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
2608   def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
2609   def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
2610   def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
2611   def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
2612   def int_aarch64_sme_st1b_vert  : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
2613   def int_aarch64_sme_st1h_vert  : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
2614   def int_aarch64_sme_st1w_vert  : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
2615   def int_aarch64_sme_st1d_vert  : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
2616   def int_aarch64_sme_st1q_vert  : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
2617
2618   // Spill + fill
2619   def int_aarch64_sme_ldr : DefaultAttrsIntrinsic<
2620     [], [llvm_i32_ty, llvm_ptr_ty]>;
2621   def int_aarch64_sme_str : DefaultAttrsIntrinsic<
2622     [], [llvm_i32_ty, llvm_ptr_ty]>;
2623
2624   class SME_TileToVector_Intrinsic
2625       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2626           [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i64_ty, llvm_i32_ty]>;
2627   class SME_VectorToTile_Intrinsic
2628       : DefaultAttrsIntrinsic<[],
2629           [llvm_i64_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2630            llvm_anyvector_ty]>;
2631
2632   def int_aarch64_sme_read_horiz  : SME_TileToVector_Intrinsic;
2633   def int_aarch64_sme_read_vert   : SME_TileToVector_Intrinsic;
2634   def int_aarch64_sme_write_horiz : SME_VectorToTile_Intrinsic;
2635   def int_aarch64_sme_write_vert  : SME_VectorToTile_Intrinsic;
2636
2637   def int_aarch64_sme_readq_horiz  : SME_TileToVector_Intrinsic;
2638   def int_aarch64_sme_readq_vert   : SME_TileToVector_Intrinsic;
2639   def int_aarch64_sme_writeq_horiz : SME_VectorToTile_Intrinsic;
2640   def int_aarch64_sme_writeq_vert  : SME_VectorToTile_Intrinsic;
2641
2642   def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i64_ty]>;
2643
2644   class SME_OuterProduct_Intrinsic
2645       : DefaultAttrsIntrinsic<[],
2646           [llvm_i64_ty,
2647            LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2648            LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2649            LLVMMatchType<0>,
2650            llvm_anyvector_ty]>;
2651
2652   def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic;
2653   def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic;
2654
2655   def int_aarch64_sme_mopa_wide : SME_OuterProduct_Intrinsic;
2656   def int_aarch64_sme_mops_wide : SME_OuterProduct_Intrinsic;
2657
2658   def int_aarch64_sme_smopa_wide  : SME_OuterProduct_Intrinsic;
2659   def int_aarch64_sme_smops_wide  : SME_OuterProduct_Intrinsic;
2660   def int_aarch64_sme_umopa_wide  : SME_OuterProduct_Intrinsic;
2661   def int_aarch64_sme_umops_wide  : SME_OuterProduct_Intrinsic;
2662   def int_aarch64_sme_sumopa_wide : SME_OuterProduct_Intrinsic;
2663   def int_aarch64_sme_sumops_wide : SME_OuterProduct_Intrinsic;
2664   def int_aarch64_sme_usmopa_wide : SME_OuterProduct_Intrinsic;
2665   def int_aarch64_sme_usmops_wide : SME_OuterProduct_Intrinsic;
2666
2667   class SME_AddVectorToTile_Intrinsic
2668       : DefaultAttrsIntrinsic<[],
2669           [llvm_i64_ty,
2670            LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2671            LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2672            llvm_anyvector_ty]>;
2673
2674   def int_aarch64_sme_addha : SME_AddVectorToTile_Intrinsic;
2675   def int_aarch64_sme_addva : SME_AddVectorToTile_Intrinsic;
2676
2677   //
2678   // Counting elements
2679   //
2680
2681   class AdvSIMD_SME_CNTSB_Intrinsic
2682     : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem]>;
2683
2684   def int_aarch64_sme_cntsb : AdvSIMD_SME_CNTSB_Intrinsic;
2685   def int_aarch64_sme_cntsh : AdvSIMD_SME_CNTSB_Intrinsic;
2686   def int_aarch64_sme_cntsw : AdvSIMD_SME_CNTSB_Intrinsic;
2687   def int_aarch64_sme_cntsd : AdvSIMD_SME_CNTSB_Intrinsic;
2688
2689   //
2690   // PSTATE Functions
2691   //
2692
2693   def int_aarch64_sme_get_pstatesm
2694       : DefaultAttrsIntrinsic<[llvm_i64_ty], [],
2695                               [IntrReadMem, IntrInaccessibleMemOnly]>;
2696
2697   def int_aarch64_sme_get_tpidr2
2698       : DefaultAttrsIntrinsic<[llvm_i64_ty], [],
2699                               [IntrNoMem, IntrHasSideEffects]>;
2700   def int_aarch64_sme_set_tpidr2
2701       : DefaultAttrsIntrinsic<[], [llvm_i64_ty],
2702                               [IntrNoMem, IntrHasSideEffects]>;
2703   // Clamp
2704   //
2705
2706   def int_aarch64_sve_sclamp : AdvSIMD_3VectorArg_Intrinsic;
2707   def int_aarch64_sve_uclamp : AdvSIMD_3VectorArg_Intrinsic;
2708
2709   //
2710   // Reversal
2711   //
2712
2713   def int_aarch64_sve_revd : AdvSIMD_Merged1VectorArg_Intrinsic;
2714
2715   //
2716   // Predicate selection
2717   //
2718
2719   def int_aarch64_sve_psel
2720       : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2721                               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2722                                LLVMMatchType<0>, llvm_i32_ty]>;
2723 }