contrib/llvm-project/clang/include/clang/Basic/arm_mve.td

   1 //===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines the set of ACLE-specified source-level intrinsic
  10 // functions wrapping the MVE vector instruction set and scalar shift
  11 // operations.
  12 //
  13 // Refer to comments in arm_mve_defs.td for the infrastructure used in
  14 // here, and to MveEmitter.cpp for how those are used in turn to
  15 // generate code.
  16 //
  17 //===----------------------------------------------------------------------===//
  18
  19 include "arm_mve_defs.td"
  20
  21 let params = T.Usual in
  22 foreach n = [ 2, 4 ] in {
  23   def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>),
  24                              (CustomCodegen<"VST24"> n:$NumVectors,
  25                               "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>;
  26   def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>),
  27                              (CustomCodegen<"VLD24"> n:$NumVectors,
  28                               "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>;
  29 }
  30
  31 multiclass bit_op_fp<IRBuilder bitop> {
  32 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
  33    (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>;
  34 }
  35
  36 multiclass bit_op_fp_with_inv<IRBuilder bitop> {
  37 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
  38    (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>;
  39 }
  40
  41 let params = T.Signed in {
  42 def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  43                               (IRIntBase<"sadd_sat", [Vector]> $a, $b)>;
  44 def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  45                               (IRIntBase<"ssub_sat", [Vector]> $a, $b)>;
  46 let pnt = PNT_NType in {
  47   def vqaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
  48                                   (IRIntBase<"sadd_sat", [Vector]> $a, (splat $b))>;
  49   def vqsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
  50                                   (IRIntBase<"ssub_sat", [Vector]> $a, (splat $b))>;
  51 }
  52 }
  53 let params = T.Unsigned in {
  54 def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  55                                 (IRIntBase<"uadd_sat", [Vector]> $a, $b)>,
  56               NameOverride<"vqaddq">;
  57 def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  58                                 (IRIntBase<"usub_sat", [Vector]> $a, $b)>,
  59               NameOverride<"vqsubq">;
  60 let pnt = PNT_NType in {
  61   def vqaddq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
  62                                     (IRIntBase<"uadd_sat", [Vector]> $a, (splat $b))>,
  63                   NameOverride<"vqaddq_n">;
  64   def vqsubq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
  65                                     (IRIntBase<"usub_sat", [Vector]> $a, (splat $b))>,
  66                   NameOverride<"vqsubq_n">;
  67 }
  68 }
  69
  70 // Some intrinsics below are implemented not as IR fragments, but as
  71 // special-purpose IR intrinsics. This is because such a general form
  72 // (such as NEON uses) required a variable-width vector size, and we are
  73 // restricted to 128 bit. Although we can possibly get clever with lane
  74 // operations, the consequent IR representation would be very hard to
  75 // write sensibly. In particular, doubling a vector's width would be a
  76 // mess. Other intrinsics just don't translate nicely into IR.
  77 let params = T.Int in {
  78 def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>;
  79 def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  80                               (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
  81 def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  82                                (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
  83 def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>;
  84 def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>;
  85 def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>;
  86 def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>;
  87 def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>;
  88 def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>;
  89 def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  90                               (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>;
  91 def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>;
  92 def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  93                               (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
  94 def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
  95                                (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
  96 def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
  97                                       (IRInt<"vmull", [DblVector, Vector]>
  98                                        $a, $b, (unsignedflag Scalar), 0)>;
  99 def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
 100                                       (IRInt<"vmull", [DblVector, Vector]>
 101                                        $a, $b, (unsignedflag Scalar), 1)>;
 102 let pnt = PNT_NType in {
 103   def vaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 104                                  (add $a, (splat $b))>;
 105   def vsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 106                                  (sub $a, (splat $b))>;
 107   def vmulq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 108                                  (mul $a, (splat $b))>;
 109   def vhaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 110                                   (IRInt<"vhadd", [Vector]> $a, (splat $b),
 111                                   (unsignedflag Scalar))>;
 112   def vhsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 113                                   (IRInt<"vhsub", [Vector]> $a, (splat $b),
 114                                   (unsignedflag Scalar))>;
 115 }
 116 }
 117 let params = T.Signed in {
 118 def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 119                                 (IRInt<"vqdmulh", [Vector]> $a, $b)>;
 120 def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 121                                  (IRInt<"vqrdmulh", [Vector]> $a, $b)>;
 122 let pnt = PNT_NType in {
 123   def vqdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 124                                     (IRInt<"vqdmulh", [Vector]> $a, (splat $b))>;
 125   def vqrdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 126                                     (IRInt<"vqrdmulh", [Vector]> $a, (splat $b))>;
 127 }
 128 }
 129
 130 let params = T.Poly, overrideKindLetter = "p" in {
 131 def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
 132                                        (IRInt<"vmull_poly", [DblVector, Vector]>
 133                                         $a, $b, 0)>;
 134 def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
 135                                        (IRInt<"vmull_poly", [DblVector, Vector]>
 136                                         $a, $b, 1)>;
 137 }
 138
 139 let params = T.Float in {
 140 def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>,
 141             NameOverride<"vaddq">;
 142 defm vandqf: bit_op_fp<and>, NameOverride<"vandq">;
 143 defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">;
 144 defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">;
 145 defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">;
 146 defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">;
 147 def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>,
 148             NameOverride<"vsubq">;
 149 def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>,
 150             NameOverride<"vmulq">;
 151
 152 let pnt = PNT_NType in {
 153   def vaddqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 154                                   (fadd $a, (splat $b))>,
 155                 NameOverride<"vaddq_n">;
 156   def vsubqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 157                                   (fsub $a, (splat $b))>,
 158                 NameOverride<"vsubq_n">;
 159   def vmulqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
 160                                   (fmul $a, (splat $b))>,
 161                 NameOverride<"vmulq_n">;
 162 }
 163 }
 164
 165 multiclass FMA<bit add> {
 166   // FMS instructions are defined in the ArmARM as if they negate the
 167   // second multiply input.
 168   defvar m2_cg = !if(add, (id $m2), (fneg $m2));
 169
 170   defvar unpred_cg = (IRIntBase<"fma", [Vector]> $m1, m2_cg, $addend);
 171   defvar pred_cg   = (IRInt<"fma_predicated", [Vector, Predicate]>
 172                           $m1, m2_cg, $addend, $pred);
 173
 174   def q: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2),
 175                    unpred_cg>;
 176
 177   def q_m: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2,
 178                                    Predicate:$pred), pred_cg>;
 179
 180   // Only FMA has the vector/scalar variants, not FMS
 181   if add then let pnt = PNT_NType in {
 182
 183     def q_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1,
 184                                      unpromoted<Scalar>:$m2_s),
 185                      (seq (splat $m2_s):$m2, unpred_cg)>;
 186     def sq_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2,
 187                                       unpromoted<Scalar>:$addend_s),
 188                         (seq (splat $addend_s):$addend, unpred_cg)>;
 189     def q_m_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1,
 190                                        unpromoted<Scalar>:$m2_s,
 191                                        Predicate:$pred),
 192                          (seq (splat $m2_s):$m2, pred_cg)>;
 193     def sq_m_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2,
 194                                         unpromoted<Scalar>:$addend_s,
 195                                         Predicate:$pred),
 196                           (seq (splat $addend_s):$addend, pred_cg)>;
 197   }
 198 }
 199
 200 let params = T.Float in {
 201   defm vfma: FMA<1>;
 202   defm vfms: FMA<0>;
 203 }
 204
 205 let params = T.Int, pnt = PNT_NType in {
 206   def vmlaq_n: Intrinsic<
 207     Vector, (args Vector:$addend, Vector:$m1, unpromoted<Scalar>:$m2_s),
 208     (add (mul $m1, (splat $m2_s)), $addend)>;
 209   def vmlasq_n: Intrinsic<
 210     Vector, (args Vector:$m1, Vector:$m2, unpromoted<Scalar>:$addend_s),
 211     (add (mul $m1, $m2), (splat $addend_s))>;
 212
 213   def vmlaq_m_n: Intrinsic<
 214     Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred),
 215     (IRInt<"vmla_n_predicated", [Vector, Predicate]>
 216         $addend, $m1, $m2_s, $pred)>;
 217   def vmlasq_m_n: Intrinsic<
 218     Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred),
 219     (IRInt<"vmlas_n_predicated", [Vector, Predicate]>
 220         $m1, $m2, $addend_s, $pred)>;
 221 }
 222
 223 multiclass VQDMLA {
 224   def hq_n: Intrinsic<
 225     Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s),
 226     (IRInt<NAME # "h", [Vector]> $addend, $m1, $m2_s)>;
 227   def shq_n: Intrinsic<
 228     Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s),
 229     (IRInt<NAME # "sh", [Vector]> $m1, $m2, $addend_s)>;
 230
 231   def hq_m_n: Intrinsic<
 232     Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred),
 233     (IRInt<NAME # "h_predicated", [Vector, Predicate]>
 234          $addend, $m1, $m2_s, $pred)>;
 235   def shq_m_n: Intrinsic<
 236     Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred),
 237     (IRInt<NAME # "sh_predicated", [Vector, Predicate]>
 238          $m1, $m2, $addend_s, $pred)>;
 239 }
 240
 241 let params = T.Signed, pnt = PNT_NType in {
 242   defm vqdmla: VQDMLA;
 243   defm vqrdmla: VQDMLA;
 244 }
 245
 246 multiclass VQDMLAD<int exchange, int round, int subtract> {
 247   def "": Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
 248        (IRInt<"vqdmlad", [Vector]> $a, $b, $c,
 249             (u32 exchange), (u32 round), (u32 subtract))>;
 250   def _m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
 251                                   Predicate:$pred),
 252        (IRInt<"vqdmlad_predicated", [Vector, Predicate]> $a, $b, $c,
 253             (u32 exchange), (u32 round), (u32 subtract), $pred)>;
 254 }
 255 let params = T.Signed in {
 256   defm vqdmladhq:   VQDMLAD<0, 0, 0>;
 257   defm vqdmladhxq:  VQDMLAD<1, 0, 0>;
 258   defm vqdmlsdhq:   VQDMLAD<0, 0, 1>;
 259   defm vqdmlsdhxq:  VQDMLAD<1, 0, 1>;
 260   defm vqrdmladhq:  VQDMLAD<0, 1, 0>;
 261   defm vqrdmladhxq: VQDMLAD<1, 1, 0>;
 262   defm vqrdmlsdhq:  VQDMLAD<0, 1, 1>;
 263   defm vqrdmlsdhxq: VQDMLAD<1, 1, 1>;
 264 }
 265
 266 let params = !listconcat(T.Int16, T.Int32) in {
 267   let pnt = PNT_None in {
 268     def vmvnq_n: Intrinsic<Vector, (args imm_simd_vmvn:$imm),
 269                            (not (splat (Scalar $imm)))>;
 270   }
 271   defm vmvnq: IntrinsicMX<Vector, (args imm_simd_vmvn:$imm, Predicate:$pred),
 272                      (select $pred, (not (splat (Scalar $imm))), $inactive),
 273                      1, "_n", PNT_NType, PNT_None>;
 274   let pnt = PNT_NType in {
 275     def vbicq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm),
 276                            (and $v, (not (splat (Scalar $imm))))>;
 277     def vorrq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm),
 278                            (or $v, (splat (Scalar $imm)))>;
 279   }
 280   def vbicq_m_n: Intrinsic<
 281     Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred),
 282     (select $pred, (and $v, (not (splat (Scalar $imm)))), $v)>;
 283   def vorrq_m_n: Intrinsic<
 284     Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred),
 285     (select $pred, (or $v, (splat (Scalar $imm))), $v)>;
 286 }
 287
 288 let params = T.Usual in {
 289   let pnt = PNT_None in
 290     def vdupq_n: Intrinsic<Vector, (args unpromoted<Scalar>:$s), (splat $s)>;
 291
 292   defm vdupq: IntrinsicMX<
 293       Vector, (args unpromoted<Scalar>:$s, Predicate:$pred),
 294       (select $pred, (splat $s), $inactive), 1, "_n", PNT_NType, PNT_None>;
 295 }
 296
 297 multiclass vxdup_mc<dag paramsIn, dag paramsOut> {
 298   defvar UnpredInt     = IRInt<NAME, [Vector]>;
 299   defvar PredInt       = IRInt<NAME # "_predicated", [Vector, Predicate]>;
 300   defvar UnpredIntCall = !con((UnpredInt $base), paramsOut);
 301   defvar PredIntCall   = !con((PredInt $inactive, $base), paramsOut, (? $pred));
 302
 303   // Straightforward case with neither writeback nor predication
 304   let pnt = PNT_N in
 305     def q_n: Intrinsic<Vector, !con((args u32:$base), paramsIn),
 306                        (xval UnpredIntCall, 0)>;
 307
 308   // Predicated form without writeback
 309   defm q: IntrinsicMX<
 310       Vector, !con((args u32:$base), paramsIn, (? Predicate:$pred)),
 311       (xval PredIntCall, 0), 1, "_n", PNT_NType, PNT_N>;
 312
 313   // Writeback without predication
 314   let pnt = PNT_WB in
 315     def q_wb: Intrinsic<
 316       Vector, !con((args Ptr<u32>:$baseaddr), paramsIn),
 317       (seq (load $baseaddr):$base,
 318            UnpredIntCall:$pair,
 319            (store (xval $pair, 1), $baseaddr),
 320            (xval $pair, 0))>;
 321
 322   // Both writeback and predicated
 323   defm q: IntrinsicMX<
 324       Vector, !con((args Ptr<u32>:$baseaddr), paramsIn, (? Predicate:$pred)),
 325       (seq (load $baseaddr):$base,
 326            PredIntCall:$pair,
 327            (store (xval $pair, 1), $baseaddr),
 328            (xval $pair, 0)), 1, "_wb", PNT_WBType, PNT_WB>;
 329 }
 330
 331 let params = T.Unsigned in {
 332   defm vidup:  vxdup_mc<(?             imm_1248:$step), (?         $step)>;
 333   defm vddup:  vxdup_mc<(?             imm_1248:$step), (?         $step)>;
 334   defm viwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
 335   defm vdwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
 336 }
 337
 338 let params = T.Int in {
 339   def vmvnq: Intrinsic<Vector, (args Vector:$a),
 340                        (xor $a, (uint_max Vector))>;
 341   defm vmvnq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
 342     (IRInt<"mvn_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
 343   def vclzq: Intrinsic<Vector, (args Vector:$a),
 344                        (IRIntBase<"ctlz", [Vector]> $a, (i1 0))>;
 345   defm vclzq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
 346     (IRInt<"clz_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
 347 }
 348 let params = T.Signed in {
 349   def vclsq: Intrinsic<Vector, (args Vector:$a), (IRInt<"vcls", [Vector]> $a)>;
 350   defm vclsq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
 351     (IRInt<"cls_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
 352
 353   def vnegq: Intrinsic<Vector, (args Vector:$a),
 354                        (sub (zeroinit Vector), $a)>;
 355   def vabsq: Intrinsic<Vector, (args Vector:$a),
 356                        (select (icmp_slt $a, (zeroinit Vector)),
 357                                (sub (zeroinit Vector), $a), $a)>;
 358   def vqnegq: Intrinsic<Vector, (args Vector:$a),
 359                         (select (icmp_eq $a, (int_min Vector)),
 360                                 (int_max Vector),
 361                                 (sub (zeroinit Vector), $a))>;
 362   def vqabsq: Intrinsic<Vector, (args Vector:$a),
 363                         (select (icmp_sgt $a, (zeroinit Vector)), $a,
 364                                 (select (icmp_eq $a, (int_min Vector)),
 365                                         (int_max Vector),
 366                                         (sub (zeroinit Vector), $a)))>;
 367
 368   foreach name = ["qneg", "qabs"] in {
 369     defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
 370       (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive),
 371       0 /* no _x variant for saturating intrinsics */>;
 372   }
 373 }
 374 let params = !listconcat(T.Signed, T.Float) in {
 375   foreach name = ["neg", "abs"] in {
 376     defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
 377       (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
 378   }
 379 }
 380 let params = T.Float in {
 381   def vnegq_f: Intrinsic<Vector, (args Vector:$a), (fneg $a)>,
 382                NameOverride<"vnegq">;
 383   def vabsq_f: Intrinsic<Vector, (args Vector:$a),
 384                (IRIntBase<"fabs", [Vector]> $a)>, NameOverride<"vabsq">;
 385 }
 386
 387 // The bitcasting below is not overcomplicating the IR because while
 388 // Vector and UVector may be different vector types at the C level i.e.
 389 // vectors of same size signed/unsigned ints. Once they're lowered
 390 // to IR, they are just bit vectors with no sign at all, so the
 391 // bitcasts will be automatically elided by IRBuilder.
 392 multiclass predicated_bit_op_fp<string int_op> {
 393 def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
 394                                 Predicate:$pred),
 395     (bitcast (IRInt<int_op, [UVector, Predicate]>
 396                     (bitcast $a, UVector),
 397                     (bitcast $b, UVector),
 398                     $pred,
 399                     (bitcast $inactive, UVector)), Vector)>;
 400 }
 401
 402 // Plain intrinsics
 403 let params = T.Usual in {
 404 def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 405                      (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>;
 406 }
 407
 408 multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?),
 409                                   int wantXVariant = 1> {
 410   defm "" : IntrinsicMX<
 411       Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
 412       !con((IRInt<operation, [Vector, Predicate]> $a, $b),
 413            extraArgs, (? $pred, $inactive)), wantXVariant>;
 414 }
 415
 416 multiclass VectorScalarArithmetic<string operation, string basename,
 417                                   dag extraArgs = (?),
 418                                   int wantXVariant = 1> {
 419   defm "" : IntrinsicMXNameOverride<
 420       Vector, (args Vector:$a, unpromoted<Scalar>:$b, Predicate:$pred),
 421       !con((IRInt<operation, [Vector, Predicate]> $a, (splat $b)),
 422            extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n",
 423            PNT_NType, PNT_NType>;
 424 }
 425
 426 multiclass VectorVectorArithmeticBitcast<string operation> {
 427   defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b,
 428                                  Predicate:$pred),
 429                                 (bitcast (IRInt<operation, [UVector, Predicate]>
 430                                           (bitcast $a, UVector),
 431                                           (bitcast $b, UVector),
 432                                           $pred,
 433                                           (bitcast $inactive, UVector)), Vector)>;
 434 }
 435
 436 // Predicated intrinsics
 437 let params = T.Usual in {
 438   defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>;
 439   defm vaddq : VectorVectorArithmetic<"add_predicated">;
 440   defm vsubq : VectorVectorArithmetic<"sub_predicated">;
 441   defm vmulq : VectorVectorArithmetic<"mul_predicated">;
 442   defm vandq : VectorVectorArithmeticBitcast<"and_predicated">;
 443   defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">;
 444   defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">;
 445   defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">;
 446   defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">;
 447
 448   defm : VectorScalarArithmetic<"add_predicated", "vaddq">;
 449   defm : VectorScalarArithmetic<"sub_predicated", "vsubq">;
 450   defm : VectorScalarArithmetic<"mul_predicated", "vmulq">;
 451 }
 452
 453 multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?),
 454                                      int wantXVariant = 1> {
 455   defm "" : IntrinsicMX<
 456       DblVector, (args Vector:$a, Vector:$b, DblPredicate:$pred),
 457       !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, $b),
 458            extraArgs, (? $pred, $inactive)), wantXVariant>;
 459 }
 460
 461 multiclass DblVectorScalarArithmetic<string operation, string basename,
 462                                      dag extraArgs = (?),
 463                                      int wantXVariant = 1> {
 464   defm "" : IntrinsicMXNameOverride<
 465       DblVector, (args Vector:$a, unpromoted<Scalar>:$b, DblPredicate:$pred),
 466       !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, (splat $b)),
 467            extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n",
 468            PNT_NType, PNT_NType>;
 469 }
 470
 471 // Predicated intrinsics - Int types only
 472 let params = T.Int in {
 473   defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>;
 474   defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>;
 475   defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>;
 476   defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>;
 477   defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>;
 478   defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>;
 479   defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>;
 480   defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>;
 481   defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>;
 482   defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>;
 483   defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>;
 484
 485   defm : VectorScalarArithmetic<"qadd_predicated", "vqaddq", (? (unsignedflag Scalar)), 0>;
 486   defm : VectorScalarArithmetic<"hadd_predicated", "vhaddq", (? (unsignedflag Scalar))>;
 487   defm : VectorScalarArithmetic<"qsub_predicated", "vqsubq", (? (unsignedflag Scalar)), 0>;
 488   defm : VectorScalarArithmetic<"hsub_predicated", "vhsubq", (? (unsignedflag Scalar))>;
 489 }
 490 let params = T.Signed in {
 491   defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>;
 492   defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>;
 493   def vminaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred),
 494                                    (IRInt<"vmina_predicated", [UVector,Predicate]> $a, $b, $pred)>;
 495   def vmaxaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred),
 496                                    (IRInt<"vmaxa_predicated", [UVector,Predicate]> $a, $b, $pred)>;
 497
 498   defm : VectorScalarArithmetic<"qdmulh_predicated", "vqdmulhq", (?), 0>;
 499   defm : VectorScalarArithmetic<"qrdmulh_predicated", "vqrdmulhq", (?), 0>;
 500 }
 501
 502 let params = T.Poly, overrideKindLetter = "p" in {
 503   defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>;
 504   defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>;
 505 }
 506
 507 let params = [s16, s32] in {
 508   def  vqdmullbq:  Intrinsic<DblVector, (args Vector:$a, Vector:$b),
 509                              (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 0)>;
 510   def  vqdmulltq:  Intrinsic<DblVector, (args Vector:$a, Vector:$b),
 511                              (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 1)>;
 512   defm vqdmullbq:  DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 0)), 0>;
 513   defm vqdmulltq:  DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 1)), 0>;
 514
 515   let pnt = PNT_NType in {
 516     def vqdmullbq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b),
 517                                (IRInt<"vqdmull", [DblVector, Vector]>
 518                                 $a, (splat $b), 0)>;
 519     def vqdmulltq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b),
 520                                (IRInt<"vqdmull", [DblVector, Vector]>
 521                                 $a, (splat $b), 1)>;
 522   }
 523   defm vqdmullbq_n: DblVectorScalarArithmetic<"vqdmull_predicated",
 524                                               "vqdmullbq", (? (u32 0)), 0>;
 525   defm vqdmulltq_n: DblVectorScalarArithmetic<"vqdmull_predicated",
 526                                               "vqdmulltq", (? (u32 1)), 0>;
 527 }
 528
 529 // Predicated intrinsics - Float types only
 530 let params = T.Float in {
 531   defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>;
 532   defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>;
 533   def vminnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
 534                                     (IRInt<"vminnma_predicated", [Vector,Predicate]> $a, $b, $pred)>;
 535   def vmaxnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
 536                                     (IRInt<"vmaxnma_predicated", [Vector,Predicate]> $a, $b, $pred)>;
 537 }
 538
 539 multiclass Reduction<Type Accumulator, string basename, list<Type> basetypes,
 540                      bit needSign = 0,
 541                      dag postCG = (seq (id $ret)),
 542                      dag accArg = (args Accumulator:$prev),
 543                      dag preCG = (seq)> {
 544   defvar intArgsBase   = (? $prev, $vec);
 545   defvar intArgsUnpred = !con(intArgsBase,
 546                               !if(needSign, (? (unsignedflag Scalar)), (?)));
 547   defvar intArgsPred   = !con(intArgsUnpred, (? $pred));
 548   defvar intUnpred     = !setop(intArgsUnpred, IRInt<basename, basetypes>);
 549   defvar intPred       = !setop(intArgsPred, IRInt<
 550             basename#"_predicated", !listconcat(basetypes, [Predicate])>);
 551
 552   def "": Intrinsic<
 553     Accumulator, !con(accArg, (args Vector:$vec)),
 554     !con(preCG, (seq intUnpred:$ret), postCG)>;
 555   def _p: Intrinsic<
 556     Accumulator, !con(accArg, (args Vector:$vec, Predicate:$pred)),
 557     !con(preCG, (seq intPred:$ret), postCG)>;
 558 }
 559
 560 let params = T.Int in {
 561 defm vminvq: Reduction<Scalar, "minv", [Vector], 1, (seq (Scalar $ret))>;
 562 defm vmaxvq: Reduction<Scalar, "maxv", [Vector], 1, (seq (Scalar $ret))>;
 563 }
 564
 565 let params = T.Signed in {
 566 defm vminavq: Reduction<UScalar, "minav", [Vector], 0, (seq (UScalar $ret))>;
 567 defm vmaxavq: Reduction<UScalar, "maxav", [Vector], 0, (seq (UScalar $ret))>;
 568 }
 569
 570 let params = T.Float in {
 571 defm vminnmvq: Reduction<Scalar, "minnmv", [Scalar, Vector]>;
 572 defm vmaxnmvq: Reduction<Scalar, "maxnmv", [Scalar, Vector]>;
 573 defm vminnmavq: Reduction<Scalar, "minnmav", [Scalar, Vector]>;
 574 defm vmaxnmavq: Reduction<Scalar, "maxnmav", [Scalar, Vector]>;
 575 }
 576
 577 foreach half = [ "b", "t" ] in {
 578   defvar halfconst = !if(!eq(half, "b"), 0, 1);
 579
 580   let params = [f32], pnt = PNT_None in {
 581     def vcvt#half#q_f16: Intrinsic<
 582       VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a),
 583       (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>;
 584     def vcvt#half#q_m_f16: Intrinsic<
 585       VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred),
 586       (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>;
 587   } // params = [f32], pnt = PNT_None
 588
 589   let params = [f16], pnt = PNT_None in {
 590     def vcvt#half#q_f32: Intrinsic<VecOf<f32>, (args Vector:$a),
 591       (IRInt<"vcvt_widen"> $a, halfconst)>;
 592     defm vcvt#half#q: IntrinsicMX<
 593       VecOf<f32>, (args Vector:$a, PredOf<f32>:$pred),
 594       (IRInt<"vcvt_widen_predicated"> $inactive, $a, halfconst, $pred),
 595       1, "_f32">;
 596   } // params = [f16], pnt = PNT_None
 597 } // loop over half = "b", "t"
 598
 599 multiclass float_int_conversions<Type FScalar, Type IScalar, IRBuilderBase ftoi, IRBuilderBase itof> {
 600   defvar FVector = VecOf<FScalar>;
 601   defvar IVector = VecOf<IScalar>;
 602
 603   let params = [IScalar] in {
 604     let pnt = PNT_2Type in {
 605       def : Intrinsic<FVector, (args IVector:$a), (itof $a, FVector)>,
 606             NameOverride<"vcvtq_" # FScalar>;
 607     }
 608     defm vcvtq: IntrinsicMX<FVector, (args IVector:$a, Predicate:$pred),
 609         (IRInt<"vcvt_fp_int_predicated", [FVector, IVector, Predicate]>
 610             $a, (unsignedflag IScalar), $pred, $inactive),
 611         1, "_" # FScalar, PNT_2Type, PNT_2Type>;
 612   }
 613   let params = [FScalar] in {
 614     let pnt = PNT_None in {
 615       def : Intrinsic<IVector, (args FVector:$a), (ftoi $a, IVector)>,
 616             NameOverride<"vcvtq_" # IScalar>;
 617
 618       foreach suffix = ["a","n","p","m"] in
 619         def : Intrinsic<IVector, (args FVector:$a),
 620                         (IRInt<"vcvt"#suffix, [IVector, FVector]>
 621                             (unsignedflag IScalar), $a)>,
 622               NameOverride<"vcvt"#suffix#"q_" # IScalar>;
 623     }
 624     defm vcvtq: IntrinsicMX<IVector, (args FVector:$a, Predicate:$pred),
 625         (IRInt<"vcvt_fp_int_predicated", [IVector, FVector, Predicate]>
 626             $a, (unsignedflag IScalar), $pred, $inactive),
 627         1, "_" # IScalar, PNT_2Type, PNT_None>;
 628
 629     foreach suffix = ["a","n","p","m"] in {
 630       defm "vcvt"#suffix#"q" : IntrinsicMX<
 631           IVector, (args FVector:$a, Predicate:$pred),
 632           (IRInt<"vcvt"#suffix#"_predicated", [IVector, FVector, Predicate]>
 633               (unsignedflag IScalar), $inactive, $a, $pred),
 634           1, "_" # IScalar, PNT_2Type, PNT_None>;
 635     }
 636   }
 637 }
 638
 639 defm "" : float_int_conversions<f32, u32, fptoui, uitofp>;
 640 defm "" : float_int_conversions<f16, u16, fptoui, uitofp>;
 641 defm "" : float_int_conversions<f32, s32, fptosi, sitofp>;
 642 defm "" : float_int_conversions<f16, s16, fptosi, sitofp>;
 643
 644 multiclass vmovl<bit top> {
 645   let params = [s8, u8, s16, u16] in {
 646     def "": Intrinsic<DblVector, (args Vector:$a),
 647       (extend (unzip $a, top), DblVector, (unsignedflag Scalar))>;
 648     defm "": IntrinsicMX<DblVector, (args Vector:$a, DblPredicate:$pred),
 649         (IRInt<"vmovl_predicated", [DblVector, Vector, DblPredicate]>
 650          $a, (unsignedflag Scalar), top, $pred, $inactive)>;
 651   }
 652 }
 653
 654 defm vmovlbq: vmovl<0>;
 655 defm vmovltq: vmovl<1>;
 656
 657 multiclass vmovn<bit top, dag wide_result> {
 658   let params = [s16, u16, s32, u32] in {
 659     def "": Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a),
 660       (trunc wide_result, HalfVector)>;
 661     def _m: Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a,
 662                                         Predicate:$pred),
 663       (IRInt<"vmovn_predicated", [HalfVector, Vector, Predicate]>
 664           $inactive, $a, top, $pred)>;
 665   }
 666 }
 667
 668 defm vmovntq: vmovn<1, (zip (vreinterpret $inactive, Vector), $a)>;
 669 defm vmovnbq: vmovn<0,
 670    (zip $a, (vreinterpret (vrev $inactive, (bitsize Scalar)), Vector))>;
 671
 672 multiclass vqmovn<bit top, Type RetScalar> {
 673   defvar RetVector = VecOf<RetScalar>;
 674
 675   let params = [s16, u16, s32, u32] in {
 676     def : Intrinsic<
 677       RetVector, (args RetVector:$inactive, Vector:$a),
 678       (IRInt<"vqmovn", [RetVector, Vector]>
 679           $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), top)>,
 680       NameOverride<NAME>;
 681     def: Intrinsic<
 682       RetVector, (args RetVector:$inactive, Vector:$a, Predicate:$pred),
 683       (IRInt<"vqmovn_predicated", [RetVector, Vector, Predicate]>
 684           $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar),
 685           top, $pred)>,
 686       NameOverride<NAME # "_m">;
 687   }
 688 }
 689
 690 let params = [s16, s32, u16, u32] in {
 691   defm vqmovntq: vqmovn<1, HalfScalar>;
 692   defm vqmovnbq: vqmovn<0, HalfScalar>;
 693 }
 694 let params = [s16, s32] in {
 695   defm vqmovuntq: vqmovn<1, UHalfScalar>;
 696   defm vqmovunbq: vqmovn<0, UHalfScalar>;
 697 }
 698
 699 multiclass vrnd<IRIntBase ir_int, string suffix> {
 700   let params = T.Float in {
 701     def "": Intrinsic<Vector, (args Vector:$a), (ir_int $a)>;
 702     defm "": IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
 703         (IRInt<"vrint"#suffix#"_predicated", [Vector, Predicate]>
 704             $a, $pred, $inactive)>;
 705   }
 706 }
 707
 708 defm vrndq:  vrnd<IRIntBase<"trunc", [Vector]>, "z">;
 709 defm vrndmq: vrnd<IRIntBase<"floor", [Vector]>, "m">;
 710 defm vrndpq: vrnd<IRIntBase<"ceil",  [Vector]>, "p">;
 711 defm vrndaq: vrnd<IRIntBase<"round", [Vector]>, "a">;
 712 defm vrndxq: vrnd<IRIntBase<"rint",  [Vector]>, "x">;
 713 defm vrndnq: vrnd<IRInt<"vrintn",    [Vector]>, "n">;
 714
 715 multiclass compare_with_pred<string condname, dag arguments,
 716                              dag cmp, string suffix> {
 717   // Make the predicated and unpredicated versions of a single comparison.
 718   def: Intrinsic<Predicate, arguments,
 719                  (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>,
 720        NameOverride<"vcmp" # condname # "q" # suffix>;
 721   def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)),
 722                  (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>,
 723        NameOverride<"vcmp" # condname # "q_m" # suffix>;
 724 }
 725
 726 multiclass compare<string condname, IRBuilder cmpop> {
 727   // Make all four variants of a comparison: the vector/vector and
 728   // vector/scalar forms, each using compare_with_pred to make a
 729   // predicated and unpredicated version.
 730   defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb),
 731                           (cmpop $va, $vb), "">;
 732   let pnt = PNT_NType in {
 733     defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb),
 734                             (cmpop $va, (splat $sb)), "_n">;
 735   }
 736 }
 737 let params = T.Int in {
 738   defm: compare<"eq", icmp_eq>;
 739   defm: compare<"ne", icmp_ne>;
 740 }
 741 let params = T.Signed in {
 742   defm: compare<"gt", icmp_sgt>;
 743   defm: compare<"ge", icmp_sge>;
 744   defm: compare<"lt", icmp_slt>;
 745   defm: compare<"le", icmp_sle>;
 746 }
 747 let params = T.Unsigned in {
 748   defm: compare<"hi", icmp_ugt>;
 749   defm: compare<"cs", icmp_uge>;
 750 }
 751 let params = T.Float in {
 752   defm: compare<"eq", fcmp_eq>;
 753   defm: compare<"ne", fcmp_ne>;
 754   defm: compare<"gt", fcmp_gt>;
 755   defm: compare<"ge", fcmp_ge>;
 756   defm: compare<"lt", fcmp_lt>;
 757   defm: compare<"le", fcmp_le>;
 758 }
 759
 760 let params = T.Signed in {
 761   def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 762                                (select (icmp_sle $a, $b), $a, $b)>;
 763   def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 764                                (select (icmp_sge $a, $b), $a, $b)>;
 765   def vminaq: Intrinsic<UVector, (args UVector:$a, Vector:$b),
 766                                  (seq (select (icmp_slt $b, (zeroinit Vector)),
 767                                               (sub (zeroinit Vector), $b), $b):$absb,
 768                                       (select (icmp_ule $a, $absb), $a, $absb))>;
 769   def vmaxaq: Intrinsic<UVector, (args UVector:$a, Vector:$b),
 770                                  (seq (select (icmp_slt $b, (zeroinit Vector)),
 771                                               (sub (zeroinit Vector), $b), $b):$absb,
 772                                       (select (icmp_uge $a, $absb), $a, $absb))>;
 773 }
 774 let params = T.Unsigned in {
 775   def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 776                                 (select (icmp_ule $a, $b), $a, $b)>,
 777               NameOverride<"vminq">;
 778   def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 779                                 (select (icmp_uge $a, $b), $a, $b)>,
 780               NameOverride<"vmaxq">;
 781 }
 782 let params = T.Float in {
 783   def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 784                                  (IRIntBase<"minnum", [Vector]> $a, $b)>;
 785   def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 786                                  (IRIntBase<"maxnum", [Vector]> $a, $b)>;
 787   def vminnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 788                                   (IRIntBase<"minnum", [Vector]>
 789                                    (IRIntBase<"fabs", [Vector]> $a),
 790                                    (IRIntBase<"fabs", [Vector]> $b))>;
 791   def vmaxnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
 792                                   (IRIntBase<"maxnum", [Vector]>
 793                                    (IRIntBase<"fabs", [Vector]> $a),
 794                                    (IRIntBase<"fabs", [Vector]> $b))>;
 795 }
 796
 797 def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred),
 798                       (select $pred, $t, $f)> { let params = T.Usual; }
 799 def vpselq_64: Intrinsic<
 800     Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred),
 801             (bitcast (select $pred, (bitcast $t, VecOf<u32>),
 802                                     (bitcast $f, VecOf<u32>)), Vector)>,
 803     NameOverride<"vpselq"> { let params = T.All64; }
 804
 805 let params = [Void], pnt = PNT_None in {
 806
 807   multiclass vctp<Type pred, string intname> {
 808     def "": Intrinsic<pred, (args u32:$val),
 809         (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>;
 810     def _m: Intrinsic<pred, (args u32:$val, pred:$inpred),
 811         (u16 (IRInt<"pred_v2i", [pred]> (and $inpred,
 812                                          (IRIntBase<intname> $val))))>;
 813   }
 814   defm vctp8q:  vctp<PredOf<u8>,  "arm_mve_vctp8">;
 815   defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">;
 816   defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">;
 817   defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">;
 818
 819   def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred),
 820                        (xor $pred, (u16 65535))>;
 821
 822 }
 823
 824 multiclass contiguous_load<string mnemonic, PrimitiveType memtype,
 825                            list<Type> same_size, list<Type> wider> {
 826   // Intrinsics named with explicit memory and element sizes that match:
 827   // vldrbq_?8, vldrhq_?16, vldrwq_?32.
 828   let params = same_size, pnt = PNT_None in {
 829     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
 830                    (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
 831          NameOverride<mnemonic>;
 832     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
 833                                  Predicate:$pred),
 834                    (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
 835                         (CPtr<Vector> $addr), !srl(memtype.size,3),
 836                         $pred, (zeroinit Vector))>,
 837          NameOverride<mnemonic # "_z">;
 838   }
 839
 840   // Synonyms for the above, with the generic name vld1q that just means
 841   // 'memory and element sizes match', and allows convenient polymorphism with
 842   // the memory and element types covariant.
 843   let params = same_size in {
 844     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
 845                    (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
 846          NameOverride<"vld1q">;
 847     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
 848                                  Predicate:$pred),
 849                    (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
 850                         (CPtr<Vector> $addr), !srl(memtype.size,3),
 851                         $pred, (zeroinit Vector))>,
 852          NameOverride<"vld1q_z">;
 853   }
 854
 855   // Intrinsics with the memory size narrower than the vector element, so that
 856   // they load less than 128 bits of memory and sign/zero extend each loaded
 857   // value into a wider vector lane.
 858   let params = wider, pnt = PNT_None in {
 859     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
 860                    (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>>
 861                                            $addr), !srl(memtype.size,3))),
 862                            Vector, (unsignedflag Scalar))>,
 863          NameOverride<mnemonic>;
 864     def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
 865                                  Predicate:$pred),
 866                    (extend (IRIntBase<"masked_load",
 867                                       [NarrowedVecOf<memtype,Vector>,
 868                                       CPtr<NarrowedVecOf<memtype,Vector>>]>
 869                                 (CPtr<NarrowedVecOf<memtype,Vector>> $addr),
 870                                 !srl(memtype.size,3), $pred,
 871                                 (zeroinit NarrowedVecOf<memtype,Vector>)),
 872                            Vector, (unsignedflag Scalar))>,
 873          NameOverride<mnemonic # "_z">;
 874   }
 875 }
 876
 877 defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
 878 defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>;
 879 defm: contiguous_load<"vldrwq", u32, T.All32, []>;
 880
 881 multiclass contiguous_store<string mnemonic, PrimitiveType memtype,
 882                            list<Type> same_size, list<Type> wider> {
 883   // Intrinsics named with explicit memory and element sizes that match:
 884   // vstrbq_?8, vstrhq_?16, vstrwq_?32.
 885   let params = same_size in {
 886     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 887                                Vector:$value),
 888                    (store $value,
 889                           (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
 890          NameOverride<mnemonic>;
 891     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 892                                Vector:$value, Predicate:$pred),
 893                    (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
 894                         $value, (Ptr<Vector> $addr),
 895                         !srl(memtype.size,3), $pred)>,
 896          NameOverride<mnemonic # "_p">;
 897   }
 898
 899   // Synonyms for the above, with the generic name vst1q that just means
 900   // 'memory and element sizes match', and allows convenient polymorphism with
 901   // the memory and element types covariant.
 902   let params = same_size in {
 903     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 904                                Vector:$value),
 905                    (store $value,
 906                           (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
 907          NameOverride<"vst1q">;
 908     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 909                                Vector:$value, Predicate:$pred),
 910                    (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
 911                         $value, (Ptr<Vector> $addr),
 912                         !srl(memtype.size,3), $pred)>,
 913          NameOverride<"vst1q_p">;
 914   }
 915
 916   // Intrinsics with the memory size narrower than the vector element, so that
 917   // they store less than 128 bits of memory, truncating each vector lane into
 918   // a narrower value to store.
 919   let params = wider in {
 920     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 921                                Vector:$value),
 922                    (store (trunc $value, NarrowedVecOf<memtype,Vector>),
 923                           (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
 924                                    !srl(memtype.size,3)))>,
 925          NameOverride<mnemonic>;
 926     def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
 927                                Vector:$value, Predicate:$pred),
 928                    (IRIntBase<"masked_store",
 929                               [NarrowedVecOf<memtype,Vector>,
 930                                Ptr<NarrowedVecOf<memtype,Vector>>]>
 931                         (trunc $value, NarrowedVecOf<memtype,Vector>),
 932                         (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
 933                         !srl(memtype.size,3), $pred)>,
 934          NameOverride<mnemonic # "_p">;
 935   }
 936 }
 937
 938 defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
 939 defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>;
 940 defm: contiguous_store<"vstrwq", u32, T.All32, []>;
 941
 942 multiclass gather_base<list<Type> types, int size> {
 943   let params = types, pnt = PNT_None in {
 944     def _gather_base: Intrinsic<
 945       Vector, (args UVector:$addr, imm_mem7bit<size>:$offset),
 946       (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>;
 947
 948     def _gather_base_z: Intrinsic<
 949       Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred),
 950       (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]>
 951           $addr, $offset, $pred)>;
 952
 953     def _gather_base_wb: Intrinsic<
 954       Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset),
 955       (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]>
 956                (load $addr), $offset):$pair,
 957            (store (xval $pair, 1), $addr),
 958            (xval $pair, 0))>;
 959
 960     def _gather_base_wb_z: Intrinsic<
 961       Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
 962                     Predicate:$pred),
 963       (seq (IRInt<"vldr_gather_base_wb_predicated",
 964                   [Vector, UVector, Predicate]>
 965                (load $addr), $offset, $pred):$pair,
 966            (store (xval $pair, 1), $addr),
 967            (xval $pair, 0))>;
 968   }
 969 }
 970
 971 defm vldrwq: gather_base<T.All32, 4>;
 972 defm vldrdq: gather_base<T.All64, 8>;
 973
 974 multiclass scatter_base<list<Type> types, int size> {
 975   let params = types in {
 976     def _scatter_base: Intrinsic<
 977       Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data),
 978       (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>;
 979
 980     def _scatter_base_p: Intrinsic<
 981       Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data,
 982                   Predicate:$pred),
 983       (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]>
 984           $addr, $offset, $data, $pred)>;
 985
 986     def _scatter_base_wb: Intrinsic<
 987       Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data),
 988       (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]>
 989                  (load $addr), $offset, $data):$wbaddr,
 990            (store $wbaddr, $addr))>;
 991
 992     def _scatter_base_wb_p: Intrinsic<
 993       Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
 994                     Vector:$data, Predicate:$pred),
 995       (seq (IRInt<"vstr_scatter_base_wb_predicated",
 996                   [UVector, Vector, Predicate]>
 997                (load $addr), $offset, $data, $pred):$wbaddr,
 998            (store $wbaddr, $addr))>;
 999   }
1000 }
1001
1002 defm vstrwq: scatter_base<T.All32, 4>;
1003 defm vstrdq: scatter_base<T.All64, 8>;
1004
1005 multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> {
1006   let params = types in {
1007     def _gather_offset: Intrinsic<
1008       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
1009       (IRInt<"vldr_gather_offset",
1010              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
1011           $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>;
1012     def _gather_offset_z: Intrinsic<
1013       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1014                     Predicate:$pred),
1015       (IRInt<"vldr_gather_offset_predicated",
1016              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
1017           $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>;
1018   }
1019 }
1020
1021 multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype,
1022                                  int shift> {
1023   let params = types in {
1024     def _gather_shifted_offset: Intrinsic<
1025       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
1026       (IRInt<"vldr_gather_offset",
1027              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
1028           $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>;
1029     def _gather_shifted_offset_z: Intrinsic<
1030       Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1031                     Predicate:$pred),
1032       (IRInt<"vldr_gather_offset_predicated",
1033              [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
1034           $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>;
1035   }
1036 }
1037
1038 multiclass gather_offset_both<list<Type> types, PrimitiveType memtype,
1039                               int shift> {
1040   defm "": gather_offset_unshifted<types, memtype>;
1041   defm "": gather_offset_shifted<types, memtype, shift>;
1042 }
1043
1044 defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>;
1045 defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
1046 defm vldrwq: gather_offset_both<T.All32, u32, 2>;
1047 defm vldrdq: gather_offset_both<T.Int64, u64, 3>;
1048
1049 multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> {
1050   let params = types in {
1051     def _scatter_offset: Intrinsic<
1052       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1053                   Vector:$data),
1054       (IRInt<"vstr_scatter_offset",
1055              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
1056           $base, $offsets, $data, memtype.size, 0)>;
1057     def _scatter_offset_p: Intrinsic<
1058       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1059                   Vector:$data, Predicate:$pred),
1060       (IRInt<"vstr_scatter_offset_predicated",
1061              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
1062           $base, $offsets, $data, memtype.size, 0, $pred)>;
1063   }
1064 }
1065
1066 multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype,
1067                                   int shift> {
1068   let params = types in {
1069     def _scatter_shifted_offset: Intrinsic<
1070       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1071                   Vector:$data),
1072       (IRInt<"vstr_scatter_offset",
1073              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
1074           $base, $offsets, $data, memtype.size, shift)>;
1075     def _scatter_shifted_offset_p: Intrinsic<
1076       Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1077                   Vector:$data, Predicate:$pred),
1078       (IRInt<"vstr_scatter_offset_predicated",
1079              [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
1080           $base, $offsets, $data, memtype.size, shift, $pred)>;
1081   }
1082 }
1083
1084 multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype,
1085                                int shift> {
1086   defm "": scatter_offset_unshifted<types, memtype>;
1087   defm "": scatter_offset_shifted<types, memtype, shift>;
1088 }
1089
1090 defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>;
1091 defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
1092 defm vstrwq: scatter_offset_both<T.All32, u32, 2>;
1093 defm vstrdq: scatter_offset_both<T.Int64, u64, 3>;
1094
1095 let params = T.Int in {
1096   def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
1097                          (shl $v, (splat (Scalar $sh)))>;
1098   defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh,
1099                                         Predicate:$pred),
1100       (IRInt<"shl_imm_predicated", [Vector, Predicate]>
1101            $v, $sh, $pred, $inactive), 1, "_n">;
1102
1103   let pnt = PNT_NType in {
1104     def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
1105                             (immshr $v, $sh, (unsignedflag Scalar))>;
1106     defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
1107                                           Predicate:$pred),
1108         (IRInt<"shr_imm_predicated", [Vector, Predicate]>
1109              $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
1110   }
1111 }
1112
1113 let params = T.Int in {
1114   def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
1115        (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
1116   def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v,
1117                                           imm_0toNm1:$sh, Predicate:$pred),
1118        (IRInt<"vqshl_imm_predicated", [Vector, Predicate]>
1119             $v, $sh, (unsignedflag Scalar), $pred, $inactive)>;
1120
1121   let pnt = PNT_NType in {
1122     def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
1123          (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
1124     defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
1125                                            Predicate:$pred),
1126          (IRInt<"vrshr_imm_predicated", [Vector, Predicate]>
1127               $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
1128   }
1129 }
1130
1131 let params = T.Signed, pnt = PNT_NType in {
1132   def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh),
1133        (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>;
1134   def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v,
1135                                             imm_0toNm1:$sh, Predicate:$pred),
1136        (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]>
1137             $v, $sh, $pred, $inactive)>;
1138 }
1139
1140 multiclass vshll_imm<int top> {
1141   let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in {
1142     def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh),
1143         (IRInt<"vshll_imm", [DblVector, Vector]>
1144             $v, $sh, (unsignedflag Scalar), top)>;
1145     defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh,
1146                                           DblPredicate:$pred),
1147         (IRInt<"vshll_imm_predicated", [DblVector, Vector, DblPredicate]>
1148             $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">;
1149   }
1150 }
1151 defm vshllbq : vshll_imm<0>;
1152 defm vshlltq : vshll_imm<1>;
1153
1154 multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME,
1155                           dag extraargs = (?)> {
1156   defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)),
1157                          [Vector], [outtype, Vector]);
1158
1159   def q_n: Intrinsic<
1160       outtype, (args outtype:$a, Vector:$b, imm:$sh),
1161       !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>;
1162
1163   def q_m_n: Intrinsic<
1164       outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred),
1165       !con((IRInt<intname # "_predicated", intparams # [Predicate]>
1166                $a, $b, $sh), extraargs, (? $pred))>;
1167 }
1168
1169 multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> {
1170   defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>;
1171   defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>;
1172 }
1173
1174 let params = [s16, s32, u16, u32], pnt = PNT_NType in {
1175   defvar U = (unsignedflag Scalar);
1176   defm vshrn   : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>;
1177   defm vqshrn  : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>;
1178   defm vrshrn  : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>;
1179   defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>;
1180 }
1181 let params = [s16, s32], pnt = PNT_NType in {
1182   defm vqshrun  : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>;
1183   defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>;
1184 }
1185 let params = T.Int, pnt = PNT_NType in {
1186   defm vsli : DyadicImmShift<Vector, imm_0toNm1>;
1187   defm vsri : DyadicImmShift<Vector, imm_1toN>;
1188 }
1189
1190 multiclass VSHL_non_imm<string scalarSuffix, int q, int r,
1191                         PolymorphicNameType pnt_scalar_unpred = PNT_Type> {
1192   let pnt = pnt_scalar_unpred in {
1193     def scalarSuffix: Intrinsic<
1194       Vector, (args Vector:$in, s32:$sh),
1195       (IRInt<"vshl_scalar", [Vector]> $in, $sh,
1196            q, r, (unsignedflag Scalar))>;
1197   }
1198   def "_m" # scalarSuffix: Intrinsic<
1199     Vector, (args Vector:$in, s32:$sh, Predicate:$pred),
1200     (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh,
1201          q, r, (unsignedflag Scalar), $pred)>;
1202
1203   def "": Intrinsic<
1204     Vector, (args Vector:$in, SVector:$sh),
1205     (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh,
1206          q, r, (unsignedflag Scalar))>;
1207   defm "": IntrinsicMX<
1208     Vector, (args Vector:$in, SVector:$sh, Predicate:$pred),
1209     (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh,
1210          q, r, (unsignedflag Scalar), $pred, $inactive),
1211     // The saturating shift intrinsics don't have an x variant, so we
1212     // set wantXVariant to 1 iff q == 0
1213     !eq(q, 0)>;
1214 }
1215
1216 let params = T.Int in {
1217   defm vshlq   : VSHL_non_imm<"_r", 0, 0>;
1218   defm vqshlq  : VSHL_non_imm<"_r", 1, 0>;
1219   defm vrshlq  : VSHL_non_imm<"_n", 0, 1, PNT_NType>;
1220   defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>;
1221 }
1222
1223 // Base class for the scalar shift intrinsics.
1224 class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
1225   Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> {
1226   let params = [Void];
1227   let pnt = PNT_None;
1228 }
1229
1230 // Subclass that includes the machinery to take a 64-bit input apart
1231 // into halves, retrieve the two halves of a shifted output as a pair,
1232 // and glue the pieces of the pair back into an i64 for output.
1233 class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
1234    ScalarShift<argtype, shiftCountArg,
1235                (seq (u32 (lshr $value, (argtype 32))):$hi,
1236                     (u32 $value):$lo,
1237                     shiftCodeGen:$pair,
1238                     (or (shl (u64 (xval $pair, 1)), (u64 32)),
1239                              (u64 (xval $pair, 0))))>;
1240
1241 // The family of saturating/rounding scalar shifts that take an
1242 // immediate shift count. They come in matched 32- and 64-bit pairs.
1243 multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> {
1244   def "": ScalarShift<arg32, (args imm_1to32:$sh),
1245                       (IRInt<NAME> $value, $sh)>;
1246   def l:  LongScalarShift<arg64, (args imm_1to32:$sh),
1247                           (IRInt<NAME # "l"> $lo, $hi, $sh)>;
1248 }
1249 defm uqshl: ScalarSaturatingShiftImm<u32, u64>;
1250 defm urshr: ScalarSaturatingShiftImm<u32, u64>;
1251 defm sqshl: ScalarSaturatingShiftImm<s32, s64>;
1252 defm srshr: ScalarSaturatingShiftImm<s32, s64>;
1253
1254 // The family of saturating/rounding scalar shifts that take a
1255 // register shift count. They also have 32- and 64-bit forms, but the
1256 // 64-bit form also has a version that saturates to 48 bits, so the IR
1257 // intrinsic takes an extra saturation-type operand.
1258 multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> {
1259   def "":          ScalarShift<arg32, (args s32:$sh),
1260                                (IRInt<NAME> $value, $sh)>;
1261   def l:       LongScalarShift<arg64, (args s32:$sh),
1262                                (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>;
1263   def l_sat48: LongScalarShift<arg64, (args s32:$sh),
1264                                (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>;
1265 }
1266 defm uqrshl: ScalarSaturatingShiftReg<u32, u64>;
1267 defm sqrshr: ScalarSaturatingShiftReg<s32, s64>;
1268
1269 // The intrinsics for LSLL and ASRL come in 64-bit versions only, with
1270 // no saturation count.
1271 def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>;
1272 def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>;
1273
1274 multiclass vadcsbc {
1275   def q: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
1276       (seq (IRInt<NAME, [Vector]> $a, $b, (shl (load $carry), 29)):$pair,
1277            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1278            (xval $pair, 0))>;
1279   def iq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
1280       (seq (IRInt<NAME, [Vector]> $a, $b, 0):$pair,
1281            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1282            (xval $pair, 0))>;
1283   def q_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
1284                                    Ptr<uint>:$carry, Predicate:$pred),
1285       (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
1286                (shl (load $carry), 29), $pred):$pair,
1287            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1288            (xval $pair, 0))>;
1289   def iq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
1290                                     Ptr<uint>:$carry, Predicate:$pred),
1291       (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
1292                0, $pred):$pair,
1293            (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1294            (xval $pair, 0))>;
1295 }
1296 let params = T.Int32 in {
1297   defm vadc: vadcsbc;
1298   defm vsbc: vadcsbc;
1299 }
1300
1301 let params = T.Int in {
1302   def vshlcq: Intrinsic<
1303     Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm),
1304     (seq (load $ps):$s,
1305          (IRInt<"vshlc", [Vector]> $v, $s, $imm):$pair,
1306          (store (xval $pair, 0), $ps),
1307          (xval $pair, 1))>;
1308   def vshlcq_m: Intrinsic<
1309     Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm, Predicate:$pred),
1310     (seq (load $ps):$s,
1311          (IRInt<"vshlc_predicated", [Vector, Predicate]>
1312               $v, $s, $imm, $pred):$pair,
1313          (store (xval $pair, 0), $ps),
1314          (xval $pair, 1))>;
1315 }
1316
1317 multiclass VectorComplexAddPred<dag not_halving, dag angle> {
1318   def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
1319      (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>;
1320   defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
1321      (IRInt<"vcaddq_predicated", [Vector, Predicate]>
1322        not_halving, angle, $inactive, $a, $b, $pred)>;
1323 }
1324
1325 multiclass VectorComplexMulPred<dag angle> {
1326   def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
1327     (IRInt<"vcmulq", [Vector]> angle, $a, $b)>;
1328   defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
1329     (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b,
1330       $pred)>;
1331 }
1332
1333 multiclass VectorComplexMLAPred<dag angle> {
1334   def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
1335     (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>;
1336   def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
1337                                    Predicate:$pred),
1338     (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>;
1339 }
1340
1341 multiclass VectorComplexAddAngle<dag not_halving> {
1342   defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>;
1343   defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>;
1344 }
1345
1346 multiclass VectorComplexMulAngle {
1347   defm ""      : VectorComplexMulPred<(u32 0)>;
1348   defm _rot90  : VectorComplexMulPred<(u32 1)>;
1349   defm _rot180 : VectorComplexMulPred<(u32 2)>;
1350   defm _rot270 : VectorComplexMulPred<(u32 3)>;
1351 }
1352
1353 multiclass VectorComplexMLAAngle {
1354   defm ""      : VectorComplexMLAPred<(u32 0)>;
1355   defm _rot90  : VectorComplexMLAPred<(u32 1)>;
1356   defm _rot180 : VectorComplexMLAPred<(u32 2)>;
1357   defm _rot270 : VectorComplexMLAPred<(u32 3)>;
1358 }
1359
1360 let params = T.Usual in
1361 defm vcaddq : VectorComplexAddAngle<(u32 1)>;
1362
1363 let params = T.Signed in
1364 defm vhcaddq : VectorComplexAddAngle<(u32 0)>;
1365
1366 let params = T.Float in {
1367 defm vcmulq : VectorComplexMulAngle;
1368 defm vcmlaq : VectorComplexMLAAngle;
1369 }
1370
1371 multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> {
1372   def xsuffix#"q"
1373     : Intrinsic<Scalar32, (args Vector:$a, Vector:$b),
1374                           (IRInt<"vmldava", [Vector]>
1375                            (unsignedflag Scalar), subtract, exchange,
1376                            (zeroinit Scalar32), $a, $b)>;
1377   def xsuffix#"q_p"
1378     : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred),
1379                           (IRInt<"vmldava_predicated", [Vector, Predicate]>
1380                            (unsignedflag Scalar), subtract, exchange,
1381                            (zeroinit Scalar32), $a, $b, $pred)>;
1382
1383   def "a"#xsuffix#"q"
1384     : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c),
1385                           (IRInt<"vmldava", [Vector]>
1386                            (unsignedflag Scalar), subtract, exchange,
1387                            $a, $b, $c)>;
1388   def "a"#xsuffix#"q_p"
1389     : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c,
1390                                 Predicate:$pred),
1391                           (IRInt<"vmldava_predicated", [Vector, Predicate]>
1392                            (unsignedflag Scalar), subtract, exchange,
1393                            $a, $b, $c, $pred)>;
1394 }
1395
1396 class IntrSplit64<Type resty, dag args, dag codegen>
1397   : Intrinsic<resty, args,
1398               (seq (u32 (lshr $a, (u64 32))):$hi,
1399                    (u32 $a):$lo,
1400                    codegen:$pair,
1401                    (or (shl (u64 (xval $pair, 1)), (u64 32)),
1402                             (u64 (xval $pair, 0))))>;
1403
1404 class IntrSplit64ZeroInit<Type resty, dag args, dag codegen>
1405   : Intrinsic<resty, args,
1406               (seq (zeroinit u32):$hi,
1407                    (zeroinit u32):$lo,
1408                    codegen:$pair,
1409                    (or (shl (u64 (xval $pair, 1)), (u64 32)),
1410                             (u64 (xval $pair, 0))))>;
1411
1412 multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange,
1413                                       string xsuffix, string irname> {
1414   def xsuffix#"q"
1415     : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b),
1416                           (IRInt<irname, [Vector]>
1417                            (unsignedflag Scalar), subtract, exchange,
1418                            $lo, $hi, $a, $b)>;
1419   def xsuffix#"q_p"
1420     : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b,
1421                                           Predicate:$pred),
1422                           (IRInt<irname#"_predicated", [Vector, Predicate]>
1423                            (unsignedflag Scalar), subtract, exchange,
1424                            $lo, $hi, $a, $b, $pred)>;
1425
1426   def "a"#xsuffix#"q"
1427     : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c),
1428                           (IRInt<irname, [Vector]>
1429                            (unsignedflag Scalar), subtract, exchange,
1430                            $lo, $hi, $b, $c)>;
1431   def "a"#xsuffix#"q_p"
1432     : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c,
1433                                   Predicate:$pred),
1434                           (IRInt<irname#"_predicated", [Vector, Predicate]>
1435                            (unsignedflag Scalar), subtract, exchange,
1436                            $lo, $hi, $b, $c, $pred)>;
1437 }
1438
1439 multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> {
1440   defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">;
1441 }
1442
1443 multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> {
1444   defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix,
1445                                        "vrmlldavha">;
1446 }
1447
1448 multiclass VADDV<bit acc, bit pred, string intbase, Type Scalar> {
1449   defvar accArg = !if(acc, (args Scalar:$acc), (args));
1450   defvar predArg = !if(pred, (args Predicate:$pred), (args));
1451   defvar intrinsic = !if(pred,
1452       IRInt<intbase # "_predicated", [Vector, Predicate]>,
1453       IRInt<intbase, [Vector]>);
1454   defvar intCG = !con((intrinsic $v, (unsignedflag Scalar)),
1455                       !if(pred, (? $pred), (?)));
1456   defvar accCG = !if(acc, (add intCG, $acc), intCG);
1457
1458   def "": Intrinsic<Scalar, !con(accArg, (args Vector:$v), predArg), accCG>;
1459 }
1460
1461 let params = T.Int in {
1462 defm vaddvq    : VADDV<0, 0, "addv", Scalar32>;
1463 defm vaddvaq   : VADDV<1, 0, "addv", Scalar32>;
1464 defm vaddvq_p  : VADDV<0, 1, "addv", Scalar32>;
1465 defm vaddvaq_p : VADDV<1, 1, "addv", Scalar32>;
1466 }
1467
1468 let params = [s32, u32] in {
1469 defm vaddlvq    : VADDV<0, 0, "addlv", Scalar64>;
1470 defm vaddlvaq   : VADDV<1, 0, "addlv", Scalar64>;
1471 defm vaddlvq_p  : VADDV<0, 1, "addlv", Scalar64>;
1472 defm vaddlvaq_p : VADDV<1, 1, "addlv", Scalar64>;
1473 }
1474
1475 let params = T.Int in {
1476 def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c),
1477     (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>;
1478 def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c,
1479                                     Predicate:$pred),
1480     (IRInt<"vabav_predicated", [Vector, Predicate]>
1481                                (unsignedflag Scalar), $a, $b, $c, $pred)>;
1482
1483 defm vmladav  : MVEBinaryVectorHoriz32<V.False, V.False, "">;
1484 }
1485
1486 let params = T.Signed in {
1487 defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">;
1488 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">;
1489 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">;
1490 }
1491
1492 let params = [u16, s16, u32, s32] in
1493 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">;
1494
1495 let params = [s16, s32] in {
1496 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">;
1497 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">;
1498 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">;
1499 }
1500
1501 let params = T.Int32 in
1502 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">;
1503
1504 let params = [s32] in {
1505 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">;
1506 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">;
1507 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">;
1508 }
1509
1510 multiclass vrev_predicated<int revsize> {
1511   defm "" : IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
1512       (IRInt<"vrev_predicated", [Vector, Predicate]>
1513           $a, revsize, $pred, $inactive)>;
1514 }
1515
1516 let params = T.All8 in {
1517   def vrev16q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 16)>;
1518   defm vrev16q: vrev_predicated<16>;
1519 }
1520 let params = !listconcat(T.All8, T.All16) in {
1521   def vrev32q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 32)>;
1522   defm vrev32q: vrev_predicated<32>;
1523 }
1524 let params = T.Usual in {
1525   def vrev64q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 64)>;
1526   defm vrev64q: vrev_predicated<64>;
1527 }
1528
1529 foreach desttype = T.All in {
1530   // We want a vreinterpretq between every pair of supported vector types
1531   // _except_ that there shouldn't be one from a type to itself.
1532   //
1533   // So this foldl expression implements what you'd write in Python as
1534   // [srctype for srctype in T.All if srctype != desttype]
1535   let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist,
1536       !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype])))
1537   in {
1538     def "vreinterpretq_" # desttype: Intrinsic<
1539         VecOf<desttype>, (args Vector:$x), (vreinterpret $x, VecOf<desttype>)>;
1540   }
1541 }
1542
1543 let params = T.All in {
1544   let pnt = PNT_None in {
1545     def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b),
1546         (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0),
1547                              $b, 1), Vector)>;
1548     def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>;
1549   }
1550
1551   // This is the polymorphic form of vuninitializedq, which takes no type
1552   // suffix, but takes an _unevaluated_ vector parameter and returns an
1553   // uninitialized vector of the same vector type.
1554   //
1555   // This intrinsic has no _non_-polymorphic form exposed to the user. But each
1556   // separately typed version of it still has to have its own clang builtin id,
1557   // which can't be called vuninitializedq_u32 or similar because that would
1558   // collide with the explicit nullary versions above. So I'm calling them
1559   // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that
1560   // full name never appears in the header file due to the polymorphicOnly
1561   // flag, and the _polymorphic suffix is omitted from the shortened name by
1562   // the custom PolymorphicNameType here.
1563   let polymorphicOnly = 1, nonEvaluating = 1,
1564       pnt = PolymorphicNameType<1, "polymorphic"> in {
1565     def vuninitializedq_polymorphic: Intrinsic<
1566         Vector, (args Vector), (undef Vector)>;
1567   }
1568
1569   def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane),
1570                             (xelt_var $v, $lane)>;
1571   def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane),
1572                             (ielt_var $v, $e, $lane)>;
1573 }
1574
1575 foreach desttype = !listconcat(T.Int16, T.Int32, T.Float) in {
1576   defvar is_dest_float = !eq(desttype.kind, "f");
1577   defvar is_dest_unsigned = !eq(desttype.kind, "u");
1578   // First immediate operand of the LLVM intrinsic
1579   defvar unsigned_flag = !if(is_dest_float, (unsignedflag Scalar),
1580                              !if(is_dest_unsigned, V.True, V.False));
1581   // For float->int conversions _n and _x_n intrinsics are not polymorphic
1582   // because the signedness of the destination type cannot be inferred.
1583   defvar pnt_nx = !if(is_dest_float, PNT_2Type, PNT_None);
1584
1585   let params = !if(is_dest_float,
1586                    !if(!eq(desttype.size, 16), T.Int16, T.Int32),
1587                    !if(!eq(desttype.size, 16), [f16], [f32])) in {
1588     let pnt = pnt_nx in
1589       def "vcvtq_n_"#desttype : Intrinsic<VecOf<desttype>,
1590         (args Vector:$a, imm_1toN:$b),
1591         (IRInt<"vcvt_fix", [VecOf<desttype>, Vector]> unsigned_flag, $a, $b)>;
1592
1593     defm "vcvtq" : IntrinsicMX<VecOf<desttype>,
1594       (args Vector:$a, imm_1toN:$b, Predicate:$p),
1595       (IRInt<"vcvt_fix_predicated", [VecOf<desttype>, Vector, Predicate]>
1596              unsigned_flag, $inactive, $a, $b, $p),
1597       1, "_n_"#desttype, PNT_2Type, pnt_nx>;
1598   }
1599 }
1600
1601 let params = T.Usual in {
1602 let pnt = PNT_NType in
1603 def vbrsrq_n: Intrinsic<Vector, (args Vector:$a, s32:$b),
1604                         (IRInt<"vbrsr", [Vector]> $a, $b)>;
1605 defm vbrsrq : IntrinsicMX<Vector, (args Vector:$a, s32:$b, Predicate:$pred),
1606                          (IRInt<"vbrsr_predicated", [Vector, Predicate]>
1607                          $inactive, $a, $b, $pred), 1, "_n",
1608                          PNT_NType, PNT_NType>;
1609 }