1 //===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines the set of ACLE-specified source-level intrinsic
10 // functions wrapping the MVE vector instruction set and scalar shift
13 // Refer to comments in arm_mve_defs.td for the infrastructure used in
14 // here, and to MveEmitter.cpp for how those are used in turn to
17 //===----------------------------------------------------------------------===//
19 include "arm_mve_defs.td"
21 let params = T.Usual in
22 foreach n = [ 2, 4 ] in {
23 def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>),
24 (CustomCodegen<"VST24"> n:$NumVectors,
25 "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>;
26 def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>),
27 (CustomCodegen<"VLD24"> n:$NumVectors,
28 "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>;
31 multiclass bit_op_fp<IRBuilder bitop> {
32 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
33 (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>;
36 multiclass bit_op_fp_with_inv<IRBuilder bitop> {
37 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
38 (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>;
41 let params = T.Signed in {
42 def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
43 (IRIntBase<"sadd_sat", [Vector]> $a, $b)>;
44 def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
45 (IRIntBase<"ssub_sat", [Vector]> $a, $b)>;
47 let params = T.Unsigned in {
48 def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
49 (IRIntBase<"uadd_sat", [Vector]> $a, $b)>,
50 NameOverride<"vqaddq">;
51 def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
52 (IRIntBase<"usub_sat", [Vector]> $a, $b)>,
53 NameOverride<"vqsubq">;
56 // Some intrinsics below are implemented not as IR fragments, but as
57 // special-purpose IR intrinsics. This is because such a general form
58 // (such as NEON uses) required a variable-width vector size, and we are
59 // restricted to 128 bit. Although we can possibly get clever with lane
60 // operations, the consequent IR representation would be very hard to
61 // write sensibly. In particular, doubling a vector's width would be a
62 // mess. Other intrinsics just don't translate nicely into IR.
63 let params = T.Int in {
64 def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>;
65 def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
66 (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
67 def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
68 (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
69 def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>;
70 def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>;
71 def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>;
72 def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>;
73 def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>;
74 def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>;
75 def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
76 (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>;
77 def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>;
78 def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
79 (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
80 def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
81 (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
82 def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
83 (IRInt<"vmull", [DblVector, Vector]>
84 $a, $b, (unsignedflag Scalar), 0)>;
85 def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
86 (IRInt<"vmull", [DblVector, Vector]>
87 $a, $b, (unsignedflag Scalar), 1)>;
89 let params = T.Signed in {
90 def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
91 (IRInt<"vqdmulh", [Vector]> $a, $b)>;
92 def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
93 (IRInt<"vqrdmulh", [Vector]> $a, $b)>;
96 let params = T.Poly, overrideKindLetter = "p" in {
97 def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
98 (IRInt<"vmull_poly", [DblVector, Vector]>
100 def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
101 (IRInt<"vmull_poly", [DblVector, Vector]>
105 let params = T.Float in {
106 def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>,
107 NameOverride<"vaddq">;
108 defm vandqf: bit_op_fp<and>, NameOverride<"vandq">;
109 defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">;
110 defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">;
111 defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">;
112 defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">;
113 def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>,
114 NameOverride<"vsubq">;
115 def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>,
116 NameOverride<"vmulq">;
119 // The bitcasting below is not overcomplicating the IR because while
120 // Vector and UVector may be different vector types at the C level i.e.
121 // vectors of same size signed/unsigned ints. Once they're lowered
122 // to IR, they are just bit vectors with no sign at all, so the
123 // bitcasts will be automatically elided by IRBuilder.
124 multiclass predicated_bit_op_fp<string int_op> {
125 def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
127 (bitcast (IRInt<int_op, [UVector, Predicate]>
128 (bitcast $a, UVector),
129 (bitcast $b, UVector),
131 (bitcast $inactive, UVector)), Vector)>;
135 let params = T.Usual in {
136 def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
137 (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>;
140 multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?),
141 int wantXVariant = 1> {
142 defm "" : IntrinsicMX<
143 Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
144 !con((IRInt<operation, [Vector, Predicate]> $a, $b),
145 extraArgs, (? $pred, $inactive)), wantXVariant>;
148 multiclass VectorVectorArithmeticBitcast<string operation> {
149 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b,
151 (bitcast (IRInt<operation, [UVector, Predicate]>
152 (bitcast $a, UVector),
153 (bitcast $b, UVector),
155 (bitcast $inactive, UVector)), Vector)>;
158 // Predicated intrinsics
159 let params = T.Usual in {
160 defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>;
161 defm vaddq : VectorVectorArithmetic<"add_predicated">;
162 defm vsubq : VectorVectorArithmetic<"sub_predicated">;
163 defm vmulq : VectorVectorArithmetic<"mul_predicated">;
164 defm vandq : VectorVectorArithmeticBitcast<"and_predicated">;
165 defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">;
166 defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">;
167 defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">;
168 defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">;
171 multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?)> {
172 defm "" : IntrinsicMX<
173 DblVector, (args Vector:$a, Vector:$b, Predicate:$pred),
174 !con((IRInt<operation, [DblVector, Vector, Predicate]> $a, $b),
175 extraArgs, (? $pred, $inactive))>;
178 // Predicated intrinsics - Int types only
179 let params = T.Int in {
180 defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>;
181 defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>;
182 defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>;
183 defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>;
184 defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>;
185 defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>;
186 defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>;
187 defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>;
188 defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>;
189 defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>;
190 defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>;
192 let params = T.Signed in {
193 defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>;
194 defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>;
197 let params = T.Poly, overrideKindLetter = "p" in {
198 defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>;
199 defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>;
202 // Predicated intrinsics - Float types only
203 let params = T.Float in {
204 defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>;
205 defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>;
208 let params = T.Int in {
209 def vminvq: Intrinsic<Scalar, (args Scalar:$prev, Vector:$vec),
210 (Scalar (IRInt<"minv", [Vector], 1> $prev, $vec))>;
211 def vmaxvq: Intrinsic<Scalar, (args Scalar:$prev, Vector:$vec),
212 (Scalar (IRInt<"maxv", [Vector], 1> $prev, $vec))>;
215 foreach half = [ "b", "t" ] in {
216 defvar halfconst = !if(!eq(half, "b"), 0, 1);
218 let params = [f32], pnt = PNT_None in {
219 def vcvt#half#q_f16: Intrinsic<
220 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a),
221 (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>;
222 def vcvt#half#q_m_f16: Intrinsic<
223 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred),
224 (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>;
225 } // params = [f32], pnt = PNT_None
226 } // loop over half = "b", "t"
228 multiclass compare_with_pred<string condname, dag arguments,
229 dag cmp, string suffix> {
230 // Make the predicated and unpredicated versions of a single comparison.
231 def: Intrinsic<Predicate, arguments,
232 (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>,
233 NameOverride<"vcmp" # condname # "q" # suffix>;
234 def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)),
235 (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>,
236 NameOverride<"vcmp" # condname # "q_m" # suffix>;
239 multiclass compare<string condname, IRBuilder cmpop> {
240 // Make all four variants of a comparison: the vector/vector and
241 // vector/scalar forms, each using compare_with_pred to make a
242 // predicated and unpredicated version.
243 defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb),
244 (cmpop $va, $vb), "">;
245 let pnt = PNT_NType in {
246 defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb),
247 (cmpop $va, (splat $sb)), "_n">;
250 let params = T.Int in {
251 defm: compare<"eq", icmp_eq>;
252 defm: compare<"ne", icmp_ne>;
254 let params = T.Signed in {
255 defm: compare<"gt", icmp_sgt>;
256 defm: compare<"ge", icmp_sge>;
257 defm: compare<"lt", icmp_slt>;
258 defm: compare<"le", icmp_sle>;
260 let params = T.Unsigned in {
261 defm: compare<"hi", icmp_ugt>;
262 defm: compare<"cs", icmp_uge>;
264 let params = T.Float in {
265 defm: compare<"eq", fcmp_eq>;
266 defm: compare<"ne", fcmp_ne>;
267 defm: compare<"gt", fcmp_gt>;
268 defm: compare<"ge", fcmp_ge>;
269 defm: compare<"lt", fcmp_lt>;
270 defm: compare<"le", fcmp_le>;
273 let params = T.Signed in {
274 def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
275 (select (icmp_sle $a, $b), $a, $b)>;
276 def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
277 (select (icmp_sge $a, $b), $a, $b)>;
279 let params = T.Unsigned in {
280 def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
281 (select (icmp_ule $a, $b), $a, $b)>,
282 NameOverride<"vminq">;
283 def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
284 (select (icmp_uge $a, $b), $a, $b)>,
285 NameOverride<"vmaxq">;
287 let params = T.Float in {
288 def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
289 (IRIntBase<"minnum", [Vector]> $a, $b)>;
290 def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
291 (IRIntBase<"maxnum", [Vector]> $a, $b)>;
294 def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred),
295 (select $pred, $t, $f)> { let params = T.Usual; }
296 def vpselq_64: Intrinsic<
297 Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred),
298 (bitcast (select $pred, (bitcast $t, VecOf<u32>),
299 (bitcast $f, VecOf<u32>)), Vector)>,
300 NameOverride<"vpselq"> { let params = T.All64; }
302 let params = [Void], pnt = PNT_None in {
304 multiclass vctp<Type pred, string intname> {
305 def "": Intrinsic<pred, (args u32:$val),
306 (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>;
307 def _m: Intrinsic<pred, (args u32:$val, pred:$inpred),
308 (u16 (IRInt<"pred_v2i", [pred]> (and $inpred,
309 (IRIntBase<intname> $val))))>;
311 defm vctp8q: vctp<PredOf<u8>, "arm_mve_vctp8">;
312 defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">;
313 defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">;
314 defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">;
316 def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred),
317 (xor $pred, (u16 65535))>;
321 multiclass contiguous_load<string mnemonic, PrimitiveType memtype,
322 list<Type> same_size, list<Type> wider> {
323 // Intrinsics named with explicit memory and element sizes that match:
324 // vldrbq_?8, vldrhq_?16, vldrwq_?32.
325 let params = same_size, pnt = PNT_None in {
326 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
327 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
328 NameOverride<mnemonic>;
329 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
331 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
332 (CPtr<Vector> $addr), !srl(memtype.size,3),
333 $pred, (zeroinit Vector))>,
334 NameOverride<mnemonic # "_z">;
337 // Synonyms for the above, with the generic name vld1q that just means
338 // 'memory and element sizes match', and allows convenient polymorphism with
339 // the memory and element types covariant.
340 let params = same_size in {
341 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
342 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
343 NameOverride<"vld1q">;
344 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
346 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
347 (CPtr<Vector> $addr), !srl(memtype.size,3),
348 $pred, (zeroinit Vector))>,
349 NameOverride<"vld1q_z">;
352 // Intrinsics with the memory size narrower than the vector element, so that
353 // they load less than 128 bits of memory and sign/zero extend each loaded
354 // value into a wider vector lane.
355 let params = wider, pnt = PNT_None in {
356 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
357 (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>>
358 $addr), !srl(memtype.size,3))),
359 Vector, (unsignedflag Scalar))>,
360 NameOverride<mnemonic>;
361 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
363 (extend (IRIntBase<"masked_load",
364 [NarrowedVecOf<memtype,Vector>,
365 CPtr<NarrowedVecOf<memtype,Vector>>]>
366 (CPtr<NarrowedVecOf<memtype,Vector>> $addr),
367 !srl(memtype.size,3), $pred,
368 (zeroinit NarrowedVecOf<memtype,Vector>)),
369 Vector, (unsignedflag Scalar))>,
370 NameOverride<mnemonic # "_z">;
374 defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
375 defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>;
376 defm: contiguous_load<"vldrwq", u32, T.All32, []>;
378 multiclass contiguous_store<string mnemonic, PrimitiveType memtype,
379 list<Type> same_size, list<Type> wider> {
380 // Intrinsics named with explicit memory and element sizes that match:
381 // vstrbq_?8, vstrhq_?16, vstrwq_?32.
382 let params = same_size in {
383 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
386 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
387 NameOverride<mnemonic>;
388 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
389 Vector:$value, Predicate:$pred),
390 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
391 $value, (Ptr<Vector> $addr),
392 !srl(memtype.size,3), $pred)>,
393 NameOverride<mnemonic # "_p">;
396 // Synonyms for the above, with the generic name vst1q that just means
397 // 'memory and element sizes match', and allows convenient polymorphism with
398 // the memory and element types covariant.
399 let params = same_size in {
400 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
403 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
404 NameOverride<"vst1q">;
405 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
406 Vector:$value, Predicate:$pred),
407 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
408 $value, (Ptr<Vector> $addr),
409 !srl(memtype.size,3), $pred)>,
410 NameOverride<"vst1q_p">;
413 // Intrinsics with the memory size narrower than the vector element, so that
414 // they store less than 128 bits of memory, truncating each vector lane into
415 // a narrower value to store.
416 let params = wider in {
417 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
419 (store (trunc $value, NarrowedVecOf<memtype,Vector>),
420 (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
421 !srl(memtype.size,3)))>,
422 NameOverride<mnemonic>;
423 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
424 Vector:$value, Predicate:$pred),
425 (IRIntBase<"masked_store",
426 [NarrowedVecOf<memtype,Vector>,
427 Ptr<NarrowedVecOf<memtype,Vector>>]>
428 (trunc $value, NarrowedVecOf<memtype,Vector>),
429 (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
430 !srl(memtype.size,3), $pred)>,
431 NameOverride<mnemonic # "_p">;
435 defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
436 defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>;
437 defm: contiguous_store<"vstrwq", u32, T.All32, []>;
439 multiclass gather_base<list<Type> types, int size> {
440 let params = types, pnt = PNT_None in {
441 def _gather_base: Intrinsic<
442 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset),
443 (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>;
445 def _gather_base_z: Intrinsic<
446 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred),
447 (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]>
448 $addr, $offset, $pred)>;
450 def _gather_base_wb: Intrinsic<
451 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset),
452 (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]>
453 (load $addr), $offset):$pair,
454 (store (xval $pair, 1), $addr),
457 def _gather_base_wb_z: Intrinsic<
458 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
460 (seq (IRInt<"vldr_gather_base_wb_predicated",
461 [Vector, UVector, Predicate]>
462 (load $addr), $offset, $pred):$pair,
463 (store (xval $pair, 1), $addr),
468 defm vldrwq: gather_base<T.All32, 4>;
469 defm vldrdq: gather_base<T.All64, 8>;
471 multiclass scatter_base<list<Type> types, int size> {
472 let params = types in {
473 def _scatter_base: Intrinsic<
474 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data),
475 (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>;
477 def _scatter_base_p: Intrinsic<
478 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data,
480 (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]>
481 $addr, $offset, $data, $pred)>;
483 def _scatter_base_wb: Intrinsic<
484 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data),
485 (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]>
486 (load $addr), $offset, $data):$wbaddr,
487 (store $wbaddr, $addr))>;
489 def _scatter_base_wb_p: Intrinsic<
490 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
491 Vector:$data, Predicate:$pred),
492 (seq (IRInt<"vstr_scatter_base_wb_predicated",
493 [UVector, Vector, Predicate]>
494 (load $addr), $offset, $data, $pred):$wbaddr,
495 (store $wbaddr, $addr))>;
499 defm vstrwq: scatter_base<T.All32, 4>;
500 defm vstrdq: scatter_base<T.All64, 8>;
502 multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> {
503 let params = types in {
504 def _gather_offset: Intrinsic<
505 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
506 (IRInt<"vldr_gather_offset",
507 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
508 $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>;
509 def _gather_offset_z: Intrinsic<
510 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
512 (IRInt<"vldr_gather_offset_predicated",
513 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
514 $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>;
518 multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype,
520 let params = types in {
521 def _gather_shifted_offset: Intrinsic<
522 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
523 (IRInt<"vldr_gather_offset",
524 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
525 $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>;
526 def _gather_shifted_offset_z: Intrinsic<
527 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
529 (IRInt<"vldr_gather_offset_predicated",
530 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
531 $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>;
535 multiclass gather_offset_both<list<Type> types, PrimitiveType memtype,
537 defm "": gather_offset_unshifted<types, memtype>;
538 defm "": gather_offset_shifted<types, memtype, shift>;
541 defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>;
542 defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
543 defm vldrwq: gather_offset_both<T.All32, u32, 2>;
544 defm vldrdq: gather_offset_both<T.Int64, u64, 3>;
546 multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> {
547 let params = types in {
548 def _scatter_offset: Intrinsic<
549 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
551 (IRInt<"vstr_scatter_offset",
552 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
553 $base, $offsets, $data, memtype.size, 0)>;
554 def _scatter_offset_p: Intrinsic<
555 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
556 Vector:$data, Predicate:$pred),
557 (IRInt<"vstr_scatter_offset_predicated",
558 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
559 $base, $offsets, $data, memtype.size, 0, $pred)>;
563 multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype,
565 let params = types in {
566 def _scatter_shifted_offset: Intrinsic<
567 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
569 (IRInt<"vstr_scatter_offset",
570 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
571 $base, $offsets, $data, memtype.size, shift)>;
572 def _scatter_shifted_offset_p: Intrinsic<
573 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
574 Vector:$data, Predicate:$pred),
575 (IRInt<"vstr_scatter_offset_predicated",
576 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
577 $base, $offsets, $data, memtype.size, shift, $pred)>;
581 multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype,
583 defm "": scatter_offset_unshifted<types, memtype>;
584 defm "": scatter_offset_shifted<types, memtype, shift>;
587 defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>;
588 defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
589 defm vstrwq: scatter_offset_both<T.All32, u32, 2>;
590 defm vstrdq: scatter_offset_both<T.Int64, u64, 3>;
592 let params = T.Int in {
593 def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
594 (shl $v, (splat (Scalar $sh)))>;
595 defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh,
597 (IRInt<"shl_imm_predicated", [Vector, Predicate]>
598 $v, $sh, $pred, $inactive), 1, "_n">;
600 let pnt = PNT_NType in {
601 def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
602 (immshr $v, $sh, (unsignedflag Scalar))>;
603 defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
605 (IRInt<"shr_imm_predicated", [Vector, Predicate]>
606 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
610 let params = T.Int in {
611 def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
612 (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
613 def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v,
614 imm_0toNm1:$sh, Predicate:$pred),
615 (IRInt<"vqshl_imm_predicated", [Vector, Predicate]>
616 $v, $sh, (unsignedflag Scalar), $pred, $inactive)>;
618 let pnt = PNT_NType in {
619 def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
620 (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
621 defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
623 (IRInt<"vrshr_imm_predicated", [Vector, Predicate]>
624 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
628 let params = T.Signed, pnt = PNT_NType in {
629 def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh),
630 (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>;
631 def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v,
632 imm_0toNm1:$sh, Predicate:$pred),
633 (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]>
634 $v, $sh, $pred, $inactive)>;
637 multiclass vshll_imm<int top> {
638 let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in {
639 def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh),
640 (IRInt<"vshll_imm", [DblVector, Vector]>
641 $v, $sh, (unsignedflag Scalar), top)>;
642 defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh,
644 (IRInt<"vshll_imm_predicated", [DblVector, Vector, Predicate]>
645 $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">;
648 defm vshllbq : vshll_imm<0>;
649 defm vshlltq : vshll_imm<1>;
651 multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME,
652 dag extraargs = (?)> {
653 defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)),
654 [Vector], [outtype, Vector]);
657 outtype, (args outtype:$a, Vector:$b, imm:$sh),
658 !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>;
660 def q_m_n: Intrinsic<
661 outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred),
662 !con((IRInt<intname # "_predicated", intparams # [Predicate]>
663 $a, $b, $sh), extraargs, (? $pred))>;
666 multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> {
667 defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>;
668 defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>;
671 let params = [s16, s32, u16, u32], pnt = PNT_NType in {
672 defvar U = (unsignedflag Scalar);
673 defm vshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>;
674 defm vqshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>;
675 defm vrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>;
676 defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>;
678 let params = [s16, s32], pnt = PNT_NType in {
679 defm vqshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>;
680 defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>;
682 let params = T.Int, pnt = PNT_NType in {
683 defm vsli : DyadicImmShift<Vector, imm_0toNm1>;
684 defm vsri : DyadicImmShift<Vector, imm_1toN>;
687 multiclass VSHL_non_imm<string scalarSuffix, int q, int r,
688 PolymorphicNameType pnt_scalar_unpred = PNT_Type> {
689 let pnt = pnt_scalar_unpred in {
690 def scalarSuffix: Intrinsic<
691 Vector, (args Vector:$in, s32:$sh),
692 (IRInt<"vshl_scalar", [Vector]> $in, $sh,
693 q, r, (unsignedflag Scalar))>;
695 def "_m" # scalarSuffix: Intrinsic<
696 Vector, (args Vector:$in, s32:$sh, Predicate:$pred),
697 (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh,
698 q, r, (unsignedflag Scalar), $pred)>;
701 Vector, (args Vector:$in, SVector:$sh),
702 (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh,
703 q, r, (unsignedflag Scalar))>;
704 defm "": IntrinsicMX<
705 Vector, (args Vector:$in, SVector:$sh, Predicate:$pred),
706 (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh,
707 q, r, (unsignedflag Scalar), $pred, $inactive),
708 // The saturating shift intrinsics don't have an x variant, so we
709 // set wantXVariant to 1 iff q == 0
713 let params = T.Int in {
714 defm vshlq : VSHL_non_imm<"_r", 0, 0>;
715 defm vqshlq : VSHL_non_imm<"_r", 1, 0>;
716 defm vrshlq : VSHL_non_imm<"_n", 0, 1, PNT_NType>;
717 defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>;
720 // Base class for the scalar shift intrinsics.
721 class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
722 Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> {
727 // Subclass that includes the machinery to take a 64-bit input apart
728 // into halves, retrieve the two halves of a shifted output as a pair,
729 // and glue the pieces of the pair back into an i64 for output.
730 class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
731 ScalarShift<argtype, shiftCountArg,
732 (seq (u32 (lshr $value, (argtype 32))):$hi,
735 (or (shl (u64 (xval $pair, 1)), (u64 32)),
736 (u64 (xval $pair, 0))))>;
738 // The family of saturating/rounding scalar shifts that take an
739 // immediate shift count. They come in matched 32- and 64-bit pairs.
740 multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> {
741 def "": ScalarShift<arg32, (args imm_1to32:$sh),
742 (IRInt<NAME> $value, $sh)>;
743 def l: LongScalarShift<arg64, (args imm_1to32:$sh),
744 (IRInt<NAME # "l"> $lo, $hi, $sh)>;
746 defm uqshl: ScalarSaturatingShiftImm<u32, u64>;
747 defm urshr: ScalarSaturatingShiftImm<u32, u64>;
748 defm sqshl: ScalarSaturatingShiftImm<s32, s64>;
749 defm srshr: ScalarSaturatingShiftImm<s32, s64>;
751 // The family of saturating/rounding scalar shifts that take a
752 // register shift count. They also have 32- and 64-bit forms, but the
753 // 64-bit form also has a version that saturates to 48 bits, so the IR
754 // intrinsic takes an extra saturation-type operand.
755 multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> {
756 def "": ScalarShift<arg32, (args s32:$sh),
757 (IRInt<NAME> $value, $sh)>;
758 def l: LongScalarShift<arg64, (args s32:$sh),
759 (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>;
760 def l_sat48: LongScalarShift<arg64, (args s32:$sh),
761 (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>;
763 defm uqrshl: ScalarSaturatingShiftReg<u32, u64>;
764 defm sqrshr: ScalarSaturatingShiftReg<s32, s64>;
766 // The intrinsics for LSLL and ASRL come in 64-bit versions only, with
767 // no saturation count.
768 def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>;
769 def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>;
771 let params = T.Int32 in {
772 def vadcq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
773 (seq (IRInt<"vadc", [Vector]> $a, $b, (shl (load $carry), 29)):$pair,
774 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
776 def vadciq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
777 (seq (IRInt<"vadc", [Vector]> $a, $b, 0):$pair,
778 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
780 def vadcq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
781 Ptr<uint>:$carry, Predicate:$pred),
782 (seq (IRInt<"vadc_predicated", [Vector, Predicate]> $inactive, $a, $b,
783 (shl (load $carry), 29), $pred):$pair,
784 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
786 def vadciq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
787 Ptr<uint>:$carry, Predicate:$pred),
788 (seq (IRInt<"vadc_predicated", [Vector, Predicate]> $inactive, $a, $b,
790 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
794 multiclass VectorComplexAddPred<dag not_halving, dag angle> {
795 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
796 (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>;
797 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
798 (IRInt<"vcaddq_predicated", [Vector, Predicate]>
799 not_halving, angle, $inactive, $a, $b, $pred)>;
802 multiclass VectorComplexMulPred<dag angle> {
803 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
804 (IRInt<"vcmulq", [Vector]> angle, $a, $b)>;
805 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
806 (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b,
810 multiclass VectorComplexMLAPred<dag angle> {
811 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
812 (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>;
813 def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
815 (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>;
818 multiclass VectorComplexAddAngle<dag not_halving> {
819 defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>;
820 defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>;
823 multiclass VectorComplexMulAngle {
824 defm "" : VectorComplexMulPred<(u32 0)>;
825 defm _rot90 : VectorComplexMulPred<(u32 1)>;
826 defm _rot180 : VectorComplexMulPred<(u32 2)>;
827 defm _rot270 : VectorComplexMulPred<(u32 3)>;
830 multiclass VectorComplexMLAAngle {
831 defm "" : VectorComplexMLAPred<(u32 0)>;
832 defm _rot90 : VectorComplexMLAPred<(u32 1)>;
833 defm _rot180 : VectorComplexMLAPred<(u32 2)>;
834 defm _rot270 : VectorComplexMLAPred<(u32 3)>;
837 let params = T.Usual in
838 defm vcaddq : VectorComplexAddAngle<(u32 1)>;
840 let params = T.Signed in
841 defm vhcaddq : VectorComplexAddAngle<(u32 0)>;
843 let params = T.Float in {
844 defm vcmulq : VectorComplexMulAngle;
845 defm vcmlaq : VectorComplexMLAAngle;
848 multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> {
850 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b),
851 (IRInt<"vmldava", [Vector]>
852 (unsignedflag Scalar), subtract, exchange,
853 (zeroinit Scalar32), $a, $b)>;
855 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred),
856 (IRInt<"vmldava_predicated", [Vector, Predicate]>
857 (unsignedflag Scalar), subtract, exchange,
858 (zeroinit Scalar32), $a, $b, $pred)>;
861 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c),
862 (IRInt<"vmldava", [Vector]>
863 (unsignedflag Scalar), subtract, exchange,
865 def "a"#xsuffix#"q_p"
866 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c,
868 (IRInt<"vmldava_predicated", [Vector, Predicate]>
869 (unsignedflag Scalar), subtract, exchange,
873 class IntrSplit64<Type resty, dag args, dag codegen>
874 : Intrinsic<resty, args,
875 (seq (u32 (lshr $a, (u64 32))):$hi,
878 (or (shl (u64 (xval $pair, 1)), (u64 32)),
879 (u64 (xval $pair, 0))))>;
881 class IntrSplit64ZeroInit<Type resty, dag args, dag codegen>
882 : Intrinsic<resty, args,
883 (seq (zeroinit u32):$hi,
886 (or (shl (u64 (xval $pair, 1)), (u64 32)),
887 (u64 (xval $pair, 0))))>;
889 multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange,
890 string xsuffix, string irname> {
892 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b),
893 (IRInt<irname, [Vector]>
894 (unsignedflag Scalar), subtract, exchange,
897 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b,
899 (IRInt<irname#"_predicated", [Vector, Predicate]>
900 (unsignedflag Scalar), subtract, exchange,
901 $lo, $hi, $a, $b, $pred)>;
904 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c),
905 (IRInt<irname, [Vector]>
906 (unsignedflag Scalar), subtract, exchange,
908 def "a"#xsuffix#"q_p"
909 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c,
911 (IRInt<irname#"_predicated", [Vector, Predicate]>
912 (unsignedflag Scalar), subtract, exchange,
913 $lo, $hi, $b, $c, $pred)>;
916 multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> {
917 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">;
920 multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> {
921 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix,
925 let params = T.Int in {
926 def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c),
927 (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>;
928 def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c,
930 (IRInt<"vabav_predicated", [Vector, Predicate]>
931 (unsignedflag Scalar), $a, $b, $c, $pred)>;
933 defm vmladav : MVEBinaryVectorHoriz32<V.False, V.False, "">;
936 let params = T.Signed in {
937 defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">;
938 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">;
939 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">;
942 let params = [u16, s16, u32, s32] in
943 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">;
945 let params = [s16, s32] in {
946 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">;
947 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">;
948 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">;
951 let params = T.Int32 in
952 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">;
954 let params = [s32] in {
955 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">;
956 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">;
957 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">;
960 foreach desttype = T.All in {
961 // We want a vreinterpretq between every pair of supported vector types
962 // _except_ that there shouldn't be one from a type to itself.
964 // So this foldl expression implements what you'd write in Python as
965 // [srctype for srctype in T.All if srctype != desttype]
966 let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist,
967 !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype])))
969 def "vreinterpretq_" # desttype: Intrinsic<
970 VecOf<desttype>, (args Vector:$x), (bitcast $x, VecOf<desttype>)>;
974 let params = T.All in {
975 let pnt = PNT_None in {
976 def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b),
977 (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0),
979 def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>;
982 // This is the polymorphic form of vuninitializedq, which takes no type
983 // suffix, but takes an _unevaluated_ vector parameter and returns an
984 // uninitialized vector of the same vector type.
986 // This intrinsic has no _non_-polymorphic form exposed to the user. But each
987 // separately typed version of it still has to have its own clang builtin id,
988 // which can't be called vuninitializedq_u32 or similar because that would
989 // collide with the explicit nullary versions above. So I'm calling them
990 // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that
991 // full name never appears in the header file due to the polymorphicOnly
992 // flag, and the _polymorphic suffix is omitted from the shortened name by
993 // the custom PolymorphicNameType here.
994 let polymorphicOnly = 1, nonEvaluating = 1,
995 pnt = PolymorphicNameType<1, "polymorphic"> in {
996 def vuninitializedq_polymorphic: Intrinsic<
997 Vector, (args Vector), (undef Vector)>;
1000 def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane),
1001 (xelt_var $v, $lane)>;
1002 def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane),
1003 (ielt_var $v, $e, $lane)>;