contrib/llvm-project/clang/include/clang/Basic/arm_mve.td

   1 //===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines the set of ACLE-specified source-level intrinsic
  10 // functions wrapping the MVE vector instruction set and scalar shift
  11 // operations.
  12 //
  13 // Refer to comments in arm_mve_defs.td for the infrastructure used in
  14 // here, and to MveEmitter.cpp for how those are used in turn to
  15 // generate code.
  16 //
  17 //===----------------------------------------------------------------------===//
  18
  19 include "arm_mve_defs.td"
  20
  21 let params = T.Usual in
  22 foreach n = [ 2, 4 ] in {
  23   def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>),
  24                              (CustomCodegen<"VST24"> n:$NumVectors,
  25                               "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>;
  26   def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>),
  27                              (CustomCodegen<"VLD24"> n:$NumVectors,
  28                               "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>;
  29 }
  30
  31 multiclass bit_op_fp<IRBuilder bitop> {
  32 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
  33    (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>;
  34 }
  35
  36 multiclass bit_op_fp_with_inv<IRBuilder bitop> {
  37 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
  38    (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>;
  39 }
  40
  41 let params = T.Signed in {
  42 def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  43                               (IRIntBase<"sadd_sat", [Vector]> $a, $b)>;
  44 def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  45                               (IRIntBase<"ssub_sat", [Vector]> $a, $b)>;
  46 }
  47 let params = T.Unsigned in {
  48 def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  49                                 (IRIntBase<"uadd_sat", [Vector]> $a, $b)>,
  50               NameOverride<"vqaddq">;
  51 def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  52                                 (IRIntBase<"usub_sat", [Vector]> $a, $b)>,
  53               NameOverride<"vqsubq">;
  54 }
  55
  56 // Some intrinsics below are implemented not as IR fragments, but as
  57 // special-purpose IR intrinsics. This is because such a general form
  58 // (such as NEON uses) required a variable-width vector size, and we are
  59 // restricted to 128 bit. Although we can possibly get clever with lane
  60 // operations, the consequent IR representation would be very hard to
  61 // write sensibly. In particular, doubling a vector's width would be a
  62 // mess. Other intrinsics just don't translate nicely into IR.
  63 let params = T.Int in {
  64 def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>;
  65 def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  66                               (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
  67 def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  68                                (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
  69 def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>;
  70 def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>;
  71 def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>;
  72 def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>;
  73 def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>;
  74 def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>;
  75 def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  76                               (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>;
  77 def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>;
  78 def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  79                               (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
  80 def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  81                                (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
  82 def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
  83                                       (IRInt<"vmull", [DblVector, Vector]>
  84                                        $a, $b, (unsignedflag Scalar), 0)>;
  85 def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
  86                                       (IRInt<"vmull", [DblVector, Vector]>
  87                                        $a, $b, (unsignedflag Scalar), 1)>;
  88 }
  89 let params = T.Signed in {
  90 def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  91                                 (IRInt<"vqdmulh", [Vector]> $a, $b)>;
  92 def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  93                                  (IRInt<"vqrdmulh", [Vector]> $a, $b)>;
  94 }
  95
  96 let params = T.Poly, overrideKindLetter = "p" in {
  97 def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
  98                                        (IRInt<"vmull_poly", [DblVector, Vector]>
  99                                         $a, $b, 0)>;
 100 def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
 101                                        (IRInt<"vmull_poly", [DblVector, Vector]>
 102                                         $a, $b, 1)>;
 103 }
 104
 105 let params = T.Float in {
 106 def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>,
 107             NameOverride<"vaddq">;
 108 defm vandqf: bit_op_fp<and>, NameOverride<"vandq">;
 109 defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">;
 110 defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">;
 111 defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">;
 112 defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">;
 113 def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>,
 114             NameOverride<"vsubq">;
 115 def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>,
 116             NameOverride<"vmulq">;
 117 }
 118
 119 // The bitcasting below is not overcomplicating the IR because while
 120 // Vector and UVector may be different vector types at the C level i.e.
 121 // vectors of same size signed/unsigned ints. Once they're lowered
 122 // to IR, they are just bit vectors with no sign at all, so the
 123 // bitcasts will be automatically elided by IRBuilder.
 124 multiclass predicated_bit_op_fp<string int_op> {
 125 def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
 126                                 Predicate:$pred),
 127     (bitcast (IRInt<int_op, [UVector, Predicate]>
 128                     (bitcast $a, UVector),
 129                     (bitcast $b, UVector),
 130                     $pred,
 131                     (bitcast $inactive, UVector)), Vector)>;
 132 }
 133
 134 // Plain intrinsics
 135 let params = T.Usual in {
 136 def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 137                      (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>;
 138 }
 139
 140 multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?),
 141                                   int wantXVariant = 1> {
 142   defm "" : IntrinsicMX<
 143       Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
 144       !con((IRInt<operation, [Vector, Predicate]> $a, $b),
 145            extraArgs, (? $pred, $inactive)), wantXVariant>;
 146 }
 147
 148 multiclass VectorVectorArithmeticBitcast<string operation> {
 149   defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b,
 150                                  Predicate:$pred),
 151                                 (bitcast (IRInt<operation, [UVector, Predicate]>
 152                                           (bitcast $a, UVector),
 153                                           (bitcast $b, UVector),
 154                                           $pred,
 155                                           (bitcast $inactive, UVector)), Vector)>;
 156 }
 157
 158 // Predicated intrinsics
 159 let params = T.Usual in {
 160   defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>;
 161   defm vaddq : VectorVectorArithmetic<"add_predicated">;
 162   defm vsubq : VectorVectorArithmetic<"sub_predicated">;
 163   defm vmulq : VectorVectorArithmetic<"mul_predicated">;
 164   defm vandq : VectorVectorArithmeticBitcast<"and_predicated">;
 165   defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">;
 166   defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">;
 167   defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">;
 168   defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">;
 169 }
 170
 171 multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?)> {
 172   defm "" : IntrinsicMX<
 173       DblVector, (args Vector:$a, Vector:$b, Predicate:$pred),
 174       !con((IRInt<operation, [DblVector, Vector, Predicate]> $a, $b),
 175            extraArgs, (? $pred, $inactive))>;
 176 }
 177
 178 // Predicated intrinsics - Int types only
 179 let params = T.Int in {
 180   defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>;
 181   defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>;
 182   defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>;
 183   defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>;
 184   defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>;
 185   defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>;
 186   defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>;
 187   defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>;
 188   defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>;
 189   defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>;
 190   defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>;
 191 }
 192 let params = T.Signed in {
 193   defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>;
 194   defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>;
 195 }
 196
 197 let params = T.Poly, overrideKindLetter = "p" in {
 198   defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>;
 199   defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>;
 200 }
 201
 202 // Predicated intrinsics - Float types only
 203 let params = T.Float in {
 204   defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>;
 205   defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>;
 206 }
 207
 208 let params = T.Int in {
 209 def vminvq: Intrinsic<Scalar, (args Scalar:$prev, Vector:$vec),
 210     (Scalar (IRInt<"minv", [Vector], 1> $prev, $vec))>;
 211 def vmaxvq: Intrinsic<Scalar, (args Scalar:$prev, Vector:$vec),
 212     (Scalar (IRInt<"maxv", [Vector], 1> $prev, $vec))>;
 213 }
 214
 215 foreach half = [ "b", "t" ] in {
 216   defvar halfconst = !if(!eq(half, "b"), 0, 1);
 217
 218   let params = [f32], pnt = PNT_None in {
 219     def vcvt#half#q_f16: Intrinsic<
 220       VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a),
 221       (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>;
 222     def vcvt#half#q_m_f16: Intrinsic<
 223       VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred),
 224       (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>;
 225   } // params = [f32], pnt = PNT_None
 226 } // loop over half = "b", "t"
 227
 228 multiclass compare_with_pred<string condname, dag arguments,
 229                              dag cmp, string suffix> {
 230   // Make the predicated and unpredicated versions of a single comparison.
 231   def: Intrinsic<Predicate, arguments,
 232                  (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>,
 233        NameOverride<"vcmp" # condname # "q" # suffix>;
 234   def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)),
 235                  (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>,
 236        NameOverride<"vcmp" # condname # "q_m" # suffix>;
 237 }
 238
 239 multiclass compare<string condname, IRBuilder cmpop> {
 240   // Make all four variants of a comparison: the vector/vector and
 241   // vector/scalar forms, each using compare_with_pred to make a
 242   // predicated and unpredicated version.
 243   defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb),
 244                           (cmpop $va, $vb), "">;
 245   let pnt = PNT_NType in {
 246     defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb),
 247                             (cmpop $va, (splat $sb)), "_n">;
 248   }
 249 }
 250 let params = T.Int in {
 251   defm: compare<"eq", icmp_eq>;
 252   defm: compare<"ne", icmp_ne>;
 253 }
 254 let params = T.Signed in {
 255   defm: compare<"gt", icmp_sgt>;
 256   defm: compare<"ge", icmp_sge>;
 257   defm: compare<"lt", icmp_slt>;
 258   defm: compare<"le", icmp_sle>;
 259 }
 260 let params = T.Unsigned in {
 261   defm: compare<"hi", icmp_ugt>;
 262   defm: compare<"cs", icmp_uge>;
 263 }
 264 let params = T.Float in {
 265   defm: compare<"eq", fcmp_eq>;
 266   defm: compare<"ne", fcmp_ne>;
 267   defm: compare<"gt", fcmp_gt>;
 268   defm: compare<"ge", fcmp_ge>;
 269   defm: compare<"lt", fcmp_lt>;
 270   defm: compare<"le", fcmp_le>;
 271 }
 272
 273 let params = T.Signed in {
 274   def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 275                                (select (icmp_sle $a, $b), $a, $b)>;
 276   def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 277                                (select (icmp_sge $a, $b), $a, $b)>;
 278 }
 279 let params = T.Unsigned in {
 280   def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 281                                 (select (icmp_ule $a, $b), $a, $b)>,
 282               NameOverride<"vminq">;
 283   def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 284                                 (select (icmp_uge $a, $b), $a, $b)>,
 285               NameOverride<"vmaxq">;
 286 }
 287 let params = T.Float in {
 288   def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 289                                (IRIntBase<"minnum", [Vector]> $a, $b)>;
 290   def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 291                                (IRIntBase<"maxnum", [Vector]> $a, $b)>;
 292 }
 293
 294 def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred),
 295                       (select $pred, $t, $f)> { let params = T.Usual; }
 296 def vpselq_64: Intrinsic<
 297     Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred),
 298             (bitcast (select $pred, (bitcast $t, VecOf<u32>),
 299                                     (bitcast $f, VecOf<u32>)), Vector)>,
 300     NameOverride<"vpselq"> { let params = T.All64; }
 301
 302 let params = [Void], pnt = PNT_None in {
 303
 304   multiclass vctp<Type pred, string intname> {
 305     def "": Intrinsic<pred, (args u32:$val),
 306         (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>;
 307     def _m: Intrinsic<pred, (args u32:$val, pred:$inpred),
 308         (u16 (IRInt<"pred_v2i", [pred]> (and $inpred,
 309                                          (IRIntBase<intname> $val))))>;
 310   }
 311   defm vctp8q:  vctp<PredOf<u8>,  "arm_mve_vctp8">;
 312   defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">;
 313   defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">;
 314   defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">;
 315
 316   def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred),
 317                        (xor $pred, (u16 65535))>;
 318
 319 }
 320
 321 multiclass contiguous_load<string mnemonic, PrimitiveType memtype,
 322                            list<Type> same_size, list<Type> wider> {
 323   // Intrinsics named with explicit memory and element sizes that match:
 324   // vldrbq_?8, vldrhq_?16, vldrwq_?32.
 325   let params = same_size, pnt = PNT_None in {
 326     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
 327                    (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
 328          NameOverride<mnemonic>;
 329     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
 330                                  Predicate:$pred),
 331                    (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
 332                         (CPtr<Vector> $addr), !srl(memtype.size,3),
 333                         $pred, (zeroinit Vector))>,
 334          NameOverride<mnemonic # "_z">;
 335   }
 336
 337   // Synonyms for the above, with the generic name vld1q that just means
 338   // 'memory and element sizes match', and allows convenient polymorphism with
 339   // the memory and element types covariant.
 340   let params = same_size in {
 341     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
 342                    (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
 343          NameOverride<"vld1q">;
 344     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
 345                                  Predicate:$pred),
 346                    (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
 347                         (CPtr<Vector> $addr), !srl(memtype.size,3),
 348                         $pred, (zeroinit Vector))>,
 349          NameOverride<"vld1q_z">;
 350   }
 351
 352   // Intrinsics with the memory size narrower than the vector element, so that
 353   // they load less than 128 bits of memory and sign/zero extend each loaded
 354   // value into a wider vector lane.
 355   let params = wider, pnt = PNT_None in {
 356     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
 357                    (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>>
 358                                            $addr), !srl(memtype.size,3))),
 359                            Vector, (unsignedflag Scalar))>,
 360          NameOverride<mnemonic>;
 361     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
 362                                  Predicate:$pred),
 363                    (extend (IRIntBase<"masked_load",
 364                                       [NarrowedVecOf<memtype,Vector>,
 365                                       CPtr<NarrowedVecOf<memtype,Vector>>]>
 366                                 (CPtr<NarrowedVecOf<memtype,Vector>> $addr),
 367                                 !srl(memtype.size,3), $pred,
 368                                 (zeroinit NarrowedVecOf<memtype,Vector>)),
 369                            Vector, (unsignedflag Scalar))>,
 370          NameOverride<mnemonic # "_z">;
 371   }
 372 }
 373
 374 defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
 375 defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>;
 376 defm: contiguous_load<"vldrwq", u32, T.All32, []>;
 377
 378 multiclass contiguous_store<string mnemonic, PrimitiveType memtype,
 379                            list<Type> same_size, list<Type> wider> {
 380   // Intrinsics named with explicit memory and element sizes that match:
 381   // vstrbq_?8, vstrhq_?16, vstrwq_?32.
 382   let params = same_size in {
 383     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 384                                Vector:$value),
 385                    (store $value,
 386                           (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
 387          NameOverride<mnemonic>;
 388     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 389                                Vector:$value, Predicate:$pred),
 390                    (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
 391                         $value, (Ptr<Vector> $addr),
 392                         !srl(memtype.size,3), $pred)>,
 393          NameOverride<mnemonic # "_p">;
 394   }
 395
 396   // Synonyms for the above, with the generic name vst1q that just means
 397   // 'memory and element sizes match', and allows convenient polymorphism with
 398   // the memory and element types covariant.
 399   let params = same_size in {
 400     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 401                                Vector:$value),
 402                    (store $value,
 403                           (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
 404          NameOverride<"vst1q">;
 405     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 406                                Vector:$value, Predicate:$pred),
 407                    (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
 408                         $value, (Ptr<Vector> $addr),
 409                         !srl(memtype.size,3), $pred)>,
 410          NameOverride<"vst1q_p">;
 411   }
 412
 413   // Intrinsics with the memory size narrower than the vector element, so that
 414   // they store less than 128 bits of memory, truncating each vector lane into
 415   // a narrower value to store.
 416   let params = wider in {
 417     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 418                                Vector:$value),
 419                    (store (trunc $value, NarrowedVecOf<memtype,Vector>),
 420                           (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
 421                                    !srl(memtype.size,3)))>,
 422          NameOverride<mnemonic>;
 423     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 424                                Vector:$value, Predicate:$pred),
 425                    (IRIntBase<"masked_store",
 426                               [NarrowedVecOf<memtype,Vector>,
 427                                Ptr<NarrowedVecOf<memtype,Vector>>]>
 428                         (trunc $value, NarrowedVecOf<memtype,Vector>),
 429                         (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
 430                         !srl(memtype.size,3), $pred)>,
 431          NameOverride<mnemonic # "_p">;
 432   }
 433 }
 434
 435 defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
 436 defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>;
 437 defm: contiguous_store<"vstrwq", u32, T.All32, []>;
 438
 439 multiclass gather_base<list<Type> types, int size> {
 440   let params = types, pnt = PNT_None in {
 441     def _gather_base: Intrinsic<
 442       Vector, (args UVector:$addr, imm_mem7bit<size>:$offset),
 443       (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>;
 444
 445     def _gather_base_z: Intrinsic<
 446       Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred),
 447       (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]>
 448           $addr, $offset, $pred)>;
 449
 450     def _gather_base_wb: Intrinsic<
 451       Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset),
 452       (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]>
 453                (load $addr), $offset):$pair,
 454            (store (xval $pair, 1), $addr),
 455            (xval $pair, 0))>;
 456
 457     def _gather_base_wb_z: Intrinsic<
 458       Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
 459                     Predicate:$pred),
 460       (seq (IRInt<"vldr_gather_base_wb_predicated",
 461                   [Vector, UVector, Predicate]>
 462                (load $addr), $offset, $pred):$pair,
 463            (store (xval $pair, 1), $addr),
 464            (xval $pair, 0))>;
 465   }
 466 }
 467
 468 defm vldrwq: gather_base<T.All32, 4>;
 469 defm vldrdq: gather_base<T.All64, 8>;
 470
 471 multiclass scatter_base<list<Type> types, int size> {
 472   let params = types in {
 473     def _scatter_base: Intrinsic<
 474       Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data),
 475       (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>;
 476
 477     def _scatter_base_p: Intrinsic<
 478       Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data,
 479                   Predicate:$pred),
 480       (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]>
 481           $addr, $offset, $data, $pred)>;
 482
 483     def _scatter_base_wb: Intrinsic<
 484       Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data),
 485       (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]>
 486                  (load $addr), $offset, $data):$wbaddr,
 487            (store $wbaddr, $addr))>;
 488
 489     def _scatter_base_wb_p: Intrinsic<
 490       Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
 491                     Vector:$data, Predicate:$pred),
 492       (seq (IRInt<"vstr_scatter_base_wb_predicated",
 493                   [UVector, Vector, Predicate]>
 494                (load $addr), $offset, $data, $pred):$wbaddr,
 495            (store $wbaddr, $addr))>;
 496   }
 497 }
 498
 499 defm vstrwq: scatter_base<T.All32, 4>;
 500 defm vstrdq: scatter_base<T.All64, 8>;
 501
 502 multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> {
 503   let params = types in {
 504     def _gather_offset: Intrinsic<
 505       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
 506       (IRInt<"vldr_gather_offset",
 507              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
 508           $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>;
 509     def _gather_offset_z: Intrinsic<
 510       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
 511                     Predicate:$pred),
 512       (IRInt<"vldr_gather_offset_predicated",
 513              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
 514           $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>;
 515   }
 516 }
 517
 518 multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype,
 519                                  int shift> {
 520   let params = types in {
 521     def _gather_shifted_offset: Intrinsic<
 522       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
 523       (IRInt<"vldr_gather_offset",
 524              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
 525           $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>;
 526     def _gather_shifted_offset_z: Intrinsic<
 527       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
 528                     Predicate:$pred),
 529       (IRInt<"vldr_gather_offset_predicated",
 530              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
 531           $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>;
 532   }
 533 }
 534
 535 multiclass gather_offset_both<list<Type> types, PrimitiveType memtype,
 536                               int shift> {
 537   defm "": gather_offset_unshifted<types, memtype>;
 538   defm "": gather_offset_shifted<types, memtype, shift>;
 539 }
 540
 541 defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>;
 542 defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
 543 defm vldrwq: gather_offset_both<T.All32, u32, 2>;
 544 defm vldrdq: gather_offset_both<T.Int64, u64, 3>;
 545
 546 multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> {
 547   let params = types in {
 548     def _scatter_offset: Intrinsic<
 549       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
 550                   Vector:$data),
 551       (IRInt<"vstr_scatter_offset",
 552              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
 553           $base, $offsets, $data, memtype.size, 0)>;
 554     def _scatter_offset_p: Intrinsic<
 555       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
 556                   Vector:$data, Predicate:$pred),
 557       (IRInt<"vstr_scatter_offset_predicated",
 558              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
 559           $base, $offsets, $data, memtype.size, 0, $pred)>;
 560   }
 561 }
 562
 563 multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype,
 564                                   int shift> {
 565   let params = types in {
 566     def _scatter_shifted_offset: Intrinsic<
 567       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
 568                   Vector:$data),
 569       (IRInt<"vstr_scatter_offset",
 570              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
 571           $base, $offsets, $data, memtype.size, shift)>;
 572     def _scatter_shifted_offset_p: Intrinsic<
 573       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
 574                   Vector:$data, Predicate:$pred),
 575       (IRInt<"vstr_scatter_offset_predicated",
 576              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
 577           $base, $offsets, $data, memtype.size, shift, $pred)>;
 578   }
 579 }
 580
 581 multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype,
 582                                int shift> {
 583   defm "": scatter_offset_unshifted<types, memtype>;
 584   defm "": scatter_offset_shifted<types, memtype, shift>;
 585 }
 586
 587 defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>;
 588 defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
 589 defm vstrwq: scatter_offset_both<T.All32, u32, 2>;
 590 defm vstrdq: scatter_offset_both<T.Int64, u64, 3>;
 591
 592 let params = T.Int in {
 593   def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
 594                          (shl $v, (splat (Scalar $sh)))>;
 595   defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh,
 596                                         Predicate:$pred),
 597       (IRInt<"shl_imm_predicated", [Vector, Predicate]>
 598            $v, $sh, $pred, $inactive), 1, "_n">;
 599
 600   let pnt = PNT_NType in {
 601     def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
 602                             (immshr $v, $sh, (unsignedflag Scalar))>;
 603     defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
 604                                           Predicate:$pred),
 605         (IRInt<"shr_imm_predicated", [Vector, Predicate]>
 606              $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
 607   }
 608 }
 609
 610 let params = T.Int in {
 611   def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
 612        (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
 613   def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v,
 614                                           imm_0toNm1:$sh, Predicate:$pred),
 615        (IRInt<"vqshl_imm_predicated", [Vector, Predicate]>
 616             $v, $sh, (unsignedflag Scalar), $pred, $inactive)>;
 617
 618   let pnt = PNT_NType in {
 619     def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
 620          (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
 621     defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
 622                                            Predicate:$pred),
 623          (IRInt<"vrshr_imm_predicated", [Vector, Predicate]>
 624               $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
 625   }
 626 }
 627
 628 let params = T.Signed, pnt = PNT_NType in {
 629   def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh),
 630        (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>;
 631   def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v,
 632                                             imm_0toNm1:$sh, Predicate:$pred),
 633        (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]>
 634             $v, $sh, $pred, $inactive)>;
 635 }
 636
 637 multiclass vshll_imm<int top> {
 638   let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in {
 639     def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh),
 640         (IRInt<"vshll_imm", [DblVector, Vector]>
 641             $v, $sh, (unsignedflag Scalar), top)>;
 642     defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh,
 643                                           Predicate:$pred),
 644         (IRInt<"vshll_imm_predicated", [DblVector, Vector, Predicate]>
 645             $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">;
 646   }
 647 }
 648 defm vshllbq : vshll_imm<0>;
 649 defm vshlltq : vshll_imm<1>;
 650
 651 multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME,
 652                           dag extraargs = (?)> {
 653   defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)),
 654                          [Vector], [outtype, Vector]);
 655
 656   def q_n: Intrinsic<
 657       outtype, (args outtype:$a, Vector:$b, imm:$sh),
 658       !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>;
 659
 660   def q_m_n: Intrinsic<
 661       outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred),
 662       !con((IRInt<intname # "_predicated", intparams # [Predicate]>
 663                $a, $b, $sh), extraargs, (? $pred))>;
 664 }
 665
 666 multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> {
 667   defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>;
 668   defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>;
 669 }
 670
 671 let params = [s16, s32, u16, u32], pnt = PNT_NType in {
 672   defvar U = (unsignedflag Scalar);
 673   defm vshrn   : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>;
 674   defm vqshrn  : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>;
 675   defm vrshrn  : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>;
 676   defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>;
 677 }
 678 let params = [s16, s32], pnt = PNT_NType in {
 679   defm vqshrun  : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>;
 680   defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>;
 681 }
 682 let params = T.Int, pnt = PNT_NType in {
 683   defm vsli : DyadicImmShift<Vector, imm_0toNm1>;
 684   defm vsri : DyadicImmShift<Vector, imm_1toN>;
 685 }
 686
 687 multiclass VSHL_non_imm<string scalarSuffix, int q, int r,
 688                         PolymorphicNameType pnt_scalar_unpred = PNT_Type> {
 689   let pnt = pnt_scalar_unpred in {
 690     def scalarSuffix: Intrinsic<
 691       Vector, (args Vector:$in, s32:$sh),
 692       (IRInt<"vshl_scalar", [Vector]> $in, $sh,
 693            q, r, (unsignedflag Scalar))>;
 694   }
 695   def "_m" # scalarSuffix: Intrinsic<
 696     Vector, (args Vector:$in, s32:$sh, Predicate:$pred),
 697     (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh,
 698          q, r, (unsignedflag Scalar), $pred)>;
 699
 700   def "": Intrinsic<
 701     Vector, (args Vector:$in, SVector:$sh),
 702     (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh,
 703          q, r, (unsignedflag Scalar))>;
 704   defm "": IntrinsicMX<
 705     Vector, (args Vector:$in, SVector:$sh, Predicate:$pred),
 706     (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh,
 707          q, r, (unsignedflag Scalar), $pred, $inactive),
 708     // The saturating shift intrinsics don't have an x variant, so we
 709     // set wantXVariant to 1 iff q == 0
 710     !eq(q, 0)>;
 711 }
 712
 713 let params = T.Int in {
 714   defm vshlq   : VSHL_non_imm<"_r", 0, 0>;
 715   defm vqshlq  : VSHL_non_imm<"_r", 1, 0>;
 716   defm vrshlq  : VSHL_non_imm<"_n", 0, 1, PNT_NType>;
 717   defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>;
 718 }
 719
 720 // Base class for the scalar shift intrinsics.
 721 class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
 722   Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> {
 723   let params = [Void];
 724   let pnt = PNT_None;
 725 }
 726
 727 // Subclass that includes the machinery to take a 64-bit input apart
 728 // into halves, retrieve the two halves of a shifted output as a pair,
 729 // and glue the pieces of the pair back into an i64 for output.
 730 class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
 731    ScalarShift<argtype, shiftCountArg,
 732                (seq (u32 (lshr $value, (argtype 32))):$hi,
 733                     (u32 $value):$lo,
 734                     shiftCodeGen:$pair,
 735                     (or (shl (u64 (xval $pair, 1)), (u64 32)),
 736                              (u64 (xval $pair, 0))))>;
 737
 738 // The family of saturating/rounding scalar shifts that take an
 739 // immediate shift count. They come in matched 32- and 64-bit pairs.
 740 multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> {
 741   def "": ScalarShift<arg32, (args imm_1to32:$sh),
 742                       (IRInt<NAME> $value, $sh)>;
 743   def l:  LongScalarShift<arg64, (args imm_1to32:$sh),
 744                           (IRInt<NAME # "l"> $lo, $hi, $sh)>;
 745 }
 746 defm uqshl: ScalarSaturatingShiftImm<u32, u64>;
 747 defm urshr: ScalarSaturatingShiftImm<u32, u64>;
 748 defm sqshl: ScalarSaturatingShiftImm<s32, s64>;
 749 defm srshr: ScalarSaturatingShiftImm<s32, s64>;
 750
 751 // The family of saturating/rounding scalar shifts that take a
 752 // register shift count. They also have 32- and 64-bit forms, but the
 753 // 64-bit form also has a version that saturates to 48 bits, so the IR
 754 // intrinsic takes an extra saturation-type operand.
 755 multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> {
 756   def "":          ScalarShift<arg32, (args s32:$sh),
 757                                (IRInt<NAME> $value, $sh)>;
 758   def l:       LongScalarShift<arg64, (args s32:$sh),
 759                                (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>;
 760   def l_sat48: LongScalarShift<arg64, (args s32:$sh),
 761                                (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>;
 762 }
 763 defm uqrshl: ScalarSaturatingShiftReg<u32, u64>;
 764 defm sqrshr: ScalarSaturatingShiftReg<s32, s64>;
 765
 766 // The intrinsics for LSLL and ASRL come in 64-bit versions only, with
 767 // no saturation count.
 768 def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>;
 769 def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>;
 770
 771 let params = T.Int32 in {
 772 def vadcq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
 773     (seq (IRInt<"vadc", [Vector]> $a, $b, (shl (load $carry), 29)):$pair,
 774          (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
 775          (xval $pair, 0))>;
 776 def vadciq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
 777     (seq (IRInt<"vadc", [Vector]> $a, $b, 0):$pair,
 778          (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
 779          (xval $pair, 0))>;
 780 def vadcq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
 781                                      Ptr<uint>:$carry, Predicate:$pred),
 782     (seq (IRInt<"vadc_predicated", [Vector, Predicate]> $inactive, $a, $b,
 783              (shl (load $carry), 29), $pred):$pair,
 784          (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
 785          (xval $pair, 0))>;
 786 def vadciq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
 787                                       Ptr<uint>:$carry, Predicate:$pred),
 788     (seq (IRInt<"vadc_predicated", [Vector, Predicate]> $inactive, $a, $b,
 789              0, $pred):$pair,
 790          (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
 791          (xval $pair, 0))>;
 792 }
 793
 794 multiclass VectorComplexAddPred<dag not_halving, dag angle> {
 795   def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
 796      (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>;
 797   defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
 798      (IRInt<"vcaddq_predicated", [Vector, Predicate]>
 799        not_halving, angle, $inactive, $a, $b, $pred)>;
 800 }
 801
 802 multiclass VectorComplexMulPred<dag angle> {
 803   def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
 804     (IRInt<"vcmulq", [Vector]> angle, $a, $b)>;
 805   defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
 806     (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b,
 807       $pred)>;
 808 }
 809
 810 multiclass VectorComplexMLAPred<dag angle> {
 811   def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
 812     (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>;
 813   def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
 814                                    Predicate:$pred),
 815     (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>;
 816 }
 817
 818 multiclass VectorComplexAddAngle<dag not_halving> {
 819   defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>;
 820   defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>;
 821 }
 822
 823 multiclass VectorComplexMulAngle {
 824   defm ""      : VectorComplexMulPred<(u32 0)>;
 825   defm _rot90  : VectorComplexMulPred<(u32 1)>;
 826   defm _rot180 : VectorComplexMulPred<(u32 2)>;
 827   defm _rot270 : VectorComplexMulPred<(u32 3)>;
 828 }
 829
 830 multiclass VectorComplexMLAAngle {
 831   defm ""      : VectorComplexMLAPred<(u32 0)>;
 832   defm _rot90  : VectorComplexMLAPred<(u32 1)>;
 833   defm _rot180 : VectorComplexMLAPred<(u32 2)>;
 834   defm _rot270 : VectorComplexMLAPred<(u32 3)>;
 835 }
 836
 837 let params = T.Usual in
 838 defm vcaddq : VectorComplexAddAngle<(u32 1)>;
 839
 840 let params = T.Signed in
 841 defm vhcaddq : VectorComplexAddAngle<(u32 0)>;
 842
 843 let params = T.Float in {
 844 defm vcmulq : VectorComplexMulAngle;
 845 defm vcmlaq : VectorComplexMLAAngle;
 846 }
 847
 848 multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> {
 849   def xsuffix#"q"
 850     : Intrinsic<Scalar32, (args Vector:$a, Vector:$b),
 851                           (IRInt<"vmldava", [Vector]>
 852                            (unsignedflag Scalar), subtract, exchange,
 853                            (zeroinit Scalar32), $a, $b)>;
 854   def xsuffix#"q_p"
 855     : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred),
 856                           (IRInt<"vmldava_predicated", [Vector, Predicate]>
 857                            (unsignedflag Scalar), subtract, exchange,
 858                            (zeroinit Scalar32), $a, $b, $pred)>;
 859
 860   def "a"#xsuffix#"q"
 861     : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c),
 862                           (IRInt<"vmldava", [Vector]>
 863                            (unsignedflag Scalar), subtract, exchange,
 864                            $a, $b, $c)>;
 865   def "a"#xsuffix#"q_p"
 866     : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c,
 867                                 Predicate:$pred),
 868                           (IRInt<"vmldava_predicated", [Vector, Predicate]>
 869                            (unsignedflag Scalar), subtract, exchange,
 870                            $a, $b, $c, $pred)>;
 871 }
 872
 873 class IntrSplit64<Type resty, dag args, dag codegen>
 874   : Intrinsic<resty, args,
 875               (seq (u32 (lshr $a, (u64 32))):$hi,
 876                    (u32 $a):$lo,
 877                    codegen:$pair,
 878                    (or (shl (u64 (xval $pair, 1)), (u64 32)),
 879                             (u64 (xval $pair, 0))))>;
 880
 881 class IntrSplit64ZeroInit<Type resty, dag args, dag codegen>
 882   : Intrinsic<resty, args,
 883               (seq (zeroinit u32):$hi,
 884                    (zeroinit u32):$lo,
 885                    codegen:$pair,
 886                    (or (shl (u64 (xval $pair, 1)), (u64 32)),
 887                             (u64 (xval $pair, 0))))>;
 888
 889 multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange,
 890                                       string xsuffix, string irname> {
 891   def xsuffix#"q"
 892     : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b),
 893                           (IRInt<irname, [Vector]>
 894                            (unsignedflag Scalar), subtract, exchange,
 895                            $lo, $hi, $a, $b)>;
 896   def xsuffix#"q_p"
 897     : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b,
 898                                           Predicate:$pred),
 899                           (IRInt<irname#"_predicated", [Vector, Predicate]>
 900                            (unsignedflag Scalar), subtract, exchange,
 901                            $lo, $hi, $a, $b, $pred)>;
 902
 903   def "a"#xsuffix#"q"
 904     : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c),
 905                           (IRInt<irname, [Vector]>
 906                            (unsignedflag Scalar), subtract, exchange,
 907                            $lo, $hi, $b, $c)>;
 908   def "a"#xsuffix#"q_p"
 909     : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c,
 910                                   Predicate:$pred),
 911                           (IRInt<irname#"_predicated", [Vector, Predicate]>
 912                            (unsignedflag Scalar), subtract, exchange,
 913                            $lo, $hi, $b, $c, $pred)>;
 914 }
 915
 916 multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> {
 917   defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">;
 918 }
 919
 920 multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> {
 921   defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix,
 922                                        "vrmlldavha">;
 923 }
 924
 925 let params = T.Int in {
 926 def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c),
 927     (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>;
 928 def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c,
 929                                     Predicate:$pred),
 930     (IRInt<"vabav_predicated", [Vector, Predicate]>
 931                                (unsignedflag Scalar), $a, $b, $c, $pred)>;
 932
 933 defm vmladav  : MVEBinaryVectorHoriz32<V.False, V.False, "">;
 934 }
 935
 936 let params = T.Signed in {
 937 defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">;
 938 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">;
 939 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">;
 940 }
 941
 942 let params = [u16, s16, u32, s32] in
 943 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">;
 944
 945 let params = [s16, s32] in {
 946 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">;
 947 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">;
 948 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">;
 949 }
 950
 951 let params = T.Int32 in
 952 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">;
 953
 954 let params = [s32] in {
 955 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">;
 956 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">;
 957 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">;
 958 }
 959
 960 foreach desttype = T.All in {
 961   // We want a vreinterpretq between every pair of supported vector types
 962   // _except_ that there shouldn't be one from a type to itself.
 963   //
 964   // So this foldl expression implements what you'd write in Python as
 965   // [srctype for srctype in T.All if srctype != desttype]
 966   let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist,
 967       !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype])))
 968   in {
 969     def "vreinterpretq_" # desttype: Intrinsic<
 970         VecOf<desttype>, (args Vector:$x), (bitcast $x, VecOf<desttype>)>;
 971   }
 972 }
 973
 974 let params = T.All in {
 975   let pnt = PNT_None in {
 976     def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b),
 977         (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0),
 978                              $b, 1), Vector)>;
 979     def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>;
 980   }
 981
 982   // This is the polymorphic form of vuninitializedq, which takes no type
 983   // suffix, but takes an _unevaluated_ vector parameter and returns an
 984   // uninitialized vector of the same vector type.
 985   //
 986   // This intrinsic has no _non_-polymorphic form exposed to the user. But each
 987   // separately typed version of it still has to have its own clang builtin id,
 988   // which can't be called vuninitializedq_u32 or similar because that would
 989   // collide with the explicit nullary versions above. So I'm calling them
 990   // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that
 991   // full name never appears in the header file due to the polymorphicOnly
 992   // flag, and the _polymorphic suffix is omitted from the shortened name by
 993   // the custom PolymorphicNameType here.
 994   let polymorphicOnly = 1, nonEvaluating = 1,
 995       pnt = PolymorphicNameType<1, "polymorphic"> in {
 996     def vuninitializedq_polymorphic: Intrinsic<
 997         Vector, (args Vector), (undef Vector)>;
 998   }
 999
1000   def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane),
1001                             (xelt_var $v, $lane)>;
1002   def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane),
1003                             (ielt_var $v, $e, $lane)>;
1004 }