1 //===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines the set of ACLE-specified source-level intrinsic
10 // functions wrapping the MVE vector instruction set and scalar shift
13 // Refer to comments in arm_mve_defs.td for the infrastructure used in
14 // here, and to MveEmitter.cpp for how those are used in turn to
17 //===----------------------------------------------------------------------===//
19 include "arm_mve_defs.td"
21 let params = T.Usual in
22 foreach n = [ 2, 4 ] in {
23 def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>),
24 (CustomCodegen<"VST24"> n:$NumVectors,
25 "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>;
26 def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>),
27 (CustomCodegen<"VLD24"> n:$NumVectors,
28 "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>;
31 multiclass bit_op_fp<IRBuilder bitop> {
32 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
33 (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>;
36 multiclass bit_op_fp_with_inv<IRBuilder bitop> {
37 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
38 (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>;
41 let params = T.Signed in {
42 def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
43 (IRIntBase<"sadd_sat", [Vector]> $a, $b)>;
44 def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
45 (IRIntBase<"ssub_sat", [Vector]> $a, $b)>;
46 let pnt = PNT_NType in {
47 def vqaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
48 (IRIntBase<"sadd_sat", [Vector]> $a, (splat $b))>;
49 def vqsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
50 (IRIntBase<"ssub_sat", [Vector]> $a, (splat $b))>;
53 let params = T.Unsigned in {
54 def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
55 (IRIntBase<"uadd_sat", [Vector]> $a, $b)>,
56 NameOverride<"vqaddq">;
57 def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
58 (IRIntBase<"usub_sat", [Vector]> $a, $b)>,
59 NameOverride<"vqsubq">;
60 let pnt = PNT_NType in {
61 def vqaddq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
62 (IRIntBase<"uadd_sat", [Vector]> $a, (splat $b))>,
63 NameOverride<"vqaddq_n">;
64 def vqsubq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
65 (IRIntBase<"usub_sat", [Vector]> $a, (splat $b))>,
66 NameOverride<"vqsubq_n">;
70 // Some intrinsics below are implemented not as IR fragments, but as
71 // special-purpose IR intrinsics. This is because such a general form
72 // (such as NEON uses) required a variable-width vector size, and we are
73 // restricted to 128 bit. Although we can possibly get clever with lane
74 // operations, the consequent IR representation would be very hard to
75 // write sensibly. In particular, doubling a vector's width would be a
76 // mess. Other intrinsics just don't translate nicely into IR.
77 let params = T.Int in {
78 def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>;
79 def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
80 (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
81 def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
82 (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
83 def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>;
84 def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>;
85 def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>;
86 def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>;
87 def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>;
88 def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>;
89 def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
90 (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>;
91 def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>;
92 def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
93 (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
94 def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
95 (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
96 def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
97 (IRInt<"vmull", [DblVector, Vector]>
98 $a, $b, (unsignedflag Scalar), 0)>;
99 def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
100 (IRInt<"vmull", [DblVector, Vector]>
101 $a, $b, (unsignedflag Scalar), 1)>;
102 let pnt = PNT_NType in {
103 def vaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
104 (add $a, (splat $b))>;
105 def vsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
106 (sub $a, (splat $b))>;
107 def vmulq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
108 (mul $a, (splat $b))>;
109 def vhaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
110 (IRInt<"vhadd", [Vector]> $a, (splat $b),
111 (unsignedflag Scalar))>;
112 def vhsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
113 (IRInt<"vhsub", [Vector]> $a, (splat $b),
114 (unsignedflag Scalar))>;
117 let params = T.Signed in {
118 def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
119 (IRInt<"vqdmulh", [Vector]> $a, $b)>;
120 def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
121 (IRInt<"vqrdmulh", [Vector]> $a, $b)>;
122 let pnt = PNT_NType in {
123 def vqdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
124 (IRInt<"vqdmulh", [Vector]> $a, (splat $b))>;
125 def vqrdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
126 (IRInt<"vqrdmulh", [Vector]> $a, (splat $b))>;
130 let params = T.Poly, overrideKindLetter = "p" in {
131 def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
132 (IRInt<"vmull_poly", [DblVector, Vector]>
134 def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
135 (IRInt<"vmull_poly", [DblVector, Vector]>
139 let params = T.Float in {
140 def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>,
141 NameOverride<"vaddq">;
142 defm vandqf: bit_op_fp<and>, NameOverride<"vandq">;
143 defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">;
144 defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">;
145 defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">;
146 defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">;
147 def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>,
148 NameOverride<"vsubq">;
149 def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>,
150 NameOverride<"vmulq">;
152 let pnt = PNT_NType in {
153 def vaddqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
154 (fadd $a, (splat $b))>,
155 NameOverride<"vaddq_n">;
156 def vsubqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
157 (fsub $a, (splat $b))>,
158 NameOverride<"vsubq_n">;
159 def vmulqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
160 (fmul $a, (splat $b))>,
161 NameOverride<"vmulq_n">;
165 multiclass FMA<bit add> {
166 // FMS instructions are defined in the ArmARM as if they negate the
167 // second multiply input.
168 defvar m2_cg = !if(add, (id $m2), (fneg $m2));
170 defvar unpred_cg = (IRIntBase<"fma", [Vector]> $m1, m2_cg, $addend);
171 defvar pred_cg = (IRInt<"fma_predicated", [Vector, Predicate]>
172 $m1, m2_cg, $addend, $pred);
174 def q: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2),
177 def q_m: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2,
178 Predicate:$pred), pred_cg>;
180 // Only FMA has the vector/scalar variants, not FMS
181 if add then let pnt = PNT_NType in {
183 def q_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1,
184 unpromoted<Scalar>:$m2_s),
185 (seq (splat $m2_s):$m2, unpred_cg)>;
186 def sq_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2,
187 unpromoted<Scalar>:$addend_s),
188 (seq (splat $addend_s):$addend, unpred_cg)>;
189 def q_m_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1,
190 unpromoted<Scalar>:$m2_s,
192 (seq (splat $m2_s):$m2, pred_cg)>;
193 def sq_m_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2,
194 unpromoted<Scalar>:$addend_s,
196 (seq (splat $addend_s):$addend, pred_cg)>;
200 let params = T.Float in {
205 let params = T.Int, pnt = PNT_NType in {
206 def vmlaq_n: Intrinsic<
207 Vector, (args Vector:$addend, Vector:$m1, unpromoted<Scalar>:$m2_s),
208 (add (mul $m1, (splat $m2_s)), $addend)>;
209 def vmlasq_n: Intrinsic<
210 Vector, (args Vector:$m1, Vector:$m2, unpromoted<Scalar>:$addend_s),
211 (add (mul $m1, $m2), (splat $addend_s))>;
213 def vmlaq_m_n: Intrinsic<
214 Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred),
215 (IRInt<"vmla_n_predicated", [Vector, Predicate]>
216 $addend, $m1, $m2_s, $pred)>;
217 def vmlasq_m_n: Intrinsic<
218 Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred),
219 (IRInt<"vmlas_n_predicated", [Vector, Predicate]>
220 $m1, $m2, $addend_s, $pred)>;
225 Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s),
226 (IRInt<NAME # "h", [Vector]> $addend, $m1, $m2_s)>;
227 def shq_n: Intrinsic<
228 Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s),
229 (IRInt<NAME # "sh", [Vector]> $m1, $m2, $addend_s)>;
231 def hq_m_n: Intrinsic<
232 Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred),
233 (IRInt<NAME # "h_predicated", [Vector, Predicate]>
234 $addend, $m1, $m2_s, $pred)>;
235 def shq_m_n: Intrinsic<
236 Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred),
237 (IRInt<NAME # "sh_predicated", [Vector, Predicate]>
238 $m1, $m2, $addend_s, $pred)>;
241 let params = T.Signed, pnt = PNT_NType in {
243 defm vqrdmla: VQDMLA;
246 multiclass VQDMLAD<int exchange, int round, int subtract> {
247 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
248 (IRInt<"vqdmlad", [Vector]> $a, $b, $c,
249 (u32 exchange), (u32 round), (u32 subtract))>;
250 def _m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
252 (IRInt<"vqdmlad_predicated", [Vector, Predicate]> $a, $b, $c,
253 (u32 exchange), (u32 round), (u32 subtract), $pred)>;
255 let params = T.Signed in {
256 defm vqdmladhq: VQDMLAD<0, 0, 0>;
257 defm vqdmladhxq: VQDMLAD<1, 0, 0>;
258 defm vqdmlsdhq: VQDMLAD<0, 0, 1>;
259 defm vqdmlsdhxq: VQDMLAD<1, 0, 1>;
260 defm vqrdmladhq: VQDMLAD<0, 1, 0>;
261 defm vqrdmladhxq: VQDMLAD<1, 1, 0>;
262 defm vqrdmlsdhq: VQDMLAD<0, 1, 1>;
263 defm vqrdmlsdhxq: VQDMLAD<1, 1, 1>;
266 let params = !listconcat(T.Int16, T.Int32) in {
267 let pnt = PNT_None in {
268 def vmvnq_n: Intrinsic<Vector, (args imm_simd_vmvn:$imm),
269 (not (splat (Scalar $imm)))>;
271 defm vmvnq: IntrinsicMX<Vector, (args imm_simd_vmvn:$imm, Predicate:$pred),
272 (select $pred, (not (splat (Scalar $imm))), $inactive),
273 1, "_n", PNT_NType, PNT_None>;
274 let pnt = PNT_NType in {
275 def vbicq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm),
276 (and $v, (not (splat (Scalar $imm))))>;
277 def vorrq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm),
278 (or $v, (splat (Scalar $imm)))>;
280 def vbicq_m_n: Intrinsic<
281 Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred),
282 (select $pred, (and $v, (not (splat (Scalar $imm)))), $v)>;
283 def vorrq_m_n: Intrinsic<
284 Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred),
285 (select $pred, (or $v, (splat (Scalar $imm))), $v)>;
288 let params = T.Usual in {
289 let pnt = PNT_None in
290 def vdupq_n: Intrinsic<Vector, (args unpromoted<Scalar>:$s), (splat $s)>;
292 defm vdupq: IntrinsicMX<
293 Vector, (args unpromoted<Scalar>:$s, Predicate:$pred),
294 (select $pred, (splat $s), $inactive), 1, "_n", PNT_NType, PNT_None>;
297 multiclass vxdup_mc<dag paramsIn, dag paramsOut> {
298 defvar UnpredInt = IRInt<NAME, [Vector]>;
299 defvar PredInt = IRInt<NAME # "_predicated", [Vector, Predicate]>;
300 defvar UnpredIntCall = !con((UnpredInt $base), paramsOut);
301 defvar PredIntCall = !con((PredInt $inactive, $base), paramsOut, (? $pred));
303 // Straightforward case with neither writeback nor predication
305 def q_n: Intrinsic<Vector, !con((args u32:$base), paramsIn),
306 (xval UnpredIntCall, 0)>;
308 // Predicated form without writeback
310 Vector, !con((args u32:$base), paramsIn, (? Predicate:$pred)),
311 (xval PredIntCall, 0), 1, "_n", PNT_NType, PNT_N>;
313 // Writeback without predication
316 Vector, !con((args Ptr<u32>:$baseaddr), paramsIn),
317 (seq (load $baseaddr):$base,
319 (store (xval $pair, 1), $baseaddr),
322 // Both writeback and predicated
324 Vector, !con((args Ptr<u32>:$baseaddr), paramsIn, (? Predicate:$pred)),
325 (seq (load $baseaddr):$base,
327 (store (xval $pair, 1), $baseaddr),
328 (xval $pair, 0)), 1, "_wb", PNT_WBType, PNT_WB>;
331 let params = T.Unsigned in {
332 defm vidup: vxdup_mc<(? imm_1248:$step), (? $step)>;
333 defm vddup: vxdup_mc<(? imm_1248:$step), (? $step)>;
334 defm viwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
335 defm vdwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
338 let params = T.Int in {
339 def vmvnq: Intrinsic<Vector, (args Vector:$a),
340 (xor $a, (uint_max Vector))>;
341 defm vmvnq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
342 (IRInt<"mvn_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
343 def vclzq: Intrinsic<Vector, (args Vector:$a),
344 (IRIntBase<"ctlz", [Vector]> $a, (i1 0))>;
345 defm vclzq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
346 (IRInt<"clz_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
348 let params = T.Signed in {
349 def vclsq: Intrinsic<Vector, (args Vector:$a), (IRInt<"vcls", [Vector]> $a)>;
350 defm vclsq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
351 (IRInt<"cls_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
353 def vnegq: Intrinsic<Vector, (args Vector:$a),
354 (sub (zeroinit Vector), $a)>;
355 def vabsq: Intrinsic<Vector, (args Vector:$a),
356 (select (icmp_slt $a, (zeroinit Vector)),
357 (sub (zeroinit Vector), $a), $a)>;
358 def vqnegq: Intrinsic<Vector, (args Vector:$a),
359 (select (icmp_eq $a, (int_min Vector)),
361 (sub (zeroinit Vector), $a))>;
362 def vqabsq: Intrinsic<Vector, (args Vector:$a),
363 (select (icmp_sgt $a, (zeroinit Vector)), $a,
364 (select (icmp_eq $a, (int_min Vector)),
366 (sub (zeroinit Vector), $a)))>;
368 foreach name = ["qneg", "qabs"] in {
369 defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
370 (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive),
371 0 /* no _x variant for saturating intrinsics */>;
374 let params = !listconcat(T.Signed, T.Float) in {
375 foreach name = ["neg", "abs"] in {
376 defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
377 (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
380 let params = T.Float in {
381 def vnegq_f: Intrinsic<Vector, (args Vector:$a), (fneg $a)>,
382 NameOverride<"vnegq">;
383 def vabsq_f: Intrinsic<Vector, (args Vector:$a),
384 (IRIntBase<"fabs", [Vector]> $a)>, NameOverride<"vabsq">;
387 // The bitcasting below is not overcomplicating the IR because while
388 // Vector and UVector may be different vector types at the C level i.e.
389 // vectors of same size signed/unsigned ints. Once they're lowered
390 // to IR, they are just bit vectors with no sign at all, so the
391 // bitcasts will be automatically elided by IRBuilder.
392 multiclass predicated_bit_op_fp<string int_op> {
393 def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
395 (bitcast (IRInt<int_op, [UVector, Predicate]>
396 (bitcast $a, UVector),
397 (bitcast $b, UVector),
399 (bitcast $inactive, UVector)), Vector)>;
403 let params = T.Usual in {
404 def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
405 (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>;
408 multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?),
409 int wantXVariant = 1> {
410 defm "" : IntrinsicMX<
411 Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
412 !con((IRInt<operation, [Vector, Predicate]> $a, $b),
413 extraArgs, (? $pred, $inactive)), wantXVariant>;
416 multiclass VectorScalarArithmetic<string operation, string basename,
418 int wantXVariant = 1> {
419 defm "" : IntrinsicMXNameOverride<
420 Vector, (args Vector:$a, unpromoted<Scalar>:$b, Predicate:$pred),
421 !con((IRInt<operation, [Vector, Predicate]> $a, (splat $b)),
422 extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n",
423 PNT_NType, PNT_NType>;
426 multiclass VectorVectorArithmeticBitcast<string operation> {
427 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b,
429 (bitcast (IRInt<operation, [UVector, Predicate]>
430 (bitcast $a, UVector),
431 (bitcast $b, UVector),
433 (bitcast $inactive, UVector)), Vector)>;
436 // Predicated intrinsics
437 let params = T.Usual in {
438 defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>;
439 defm vaddq : VectorVectorArithmetic<"add_predicated">;
440 defm vsubq : VectorVectorArithmetic<"sub_predicated">;
441 defm vmulq : VectorVectorArithmetic<"mul_predicated">;
442 defm vandq : VectorVectorArithmeticBitcast<"and_predicated">;
443 defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">;
444 defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">;
445 defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">;
446 defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">;
448 defm : VectorScalarArithmetic<"add_predicated", "vaddq">;
449 defm : VectorScalarArithmetic<"sub_predicated", "vsubq">;
450 defm : VectorScalarArithmetic<"mul_predicated", "vmulq">;
453 multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?),
454 int wantXVariant = 1> {
455 defm "" : IntrinsicMX<
456 DblVector, (args Vector:$a, Vector:$b, DblPredicate:$pred),
457 !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, $b),
458 extraArgs, (? $pred, $inactive)), wantXVariant>;
461 multiclass DblVectorScalarArithmetic<string operation, string basename,
463 int wantXVariant = 1> {
464 defm "" : IntrinsicMXNameOverride<
465 DblVector, (args Vector:$a, unpromoted<Scalar>:$b, DblPredicate:$pred),
466 !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, (splat $b)),
467 extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n",
468 PNT_NType, PNT_NType>;
471 // Predicated intrinsics - Int types only
472 let params = T.Int in {
473 defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>;
474 defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>;
475 defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>;
476 defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>;
477 defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>;
478 defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>;
479 defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>;
480 defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>;
481 defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>;
482 defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>;
483 defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>;
485 defm : VectorScalarArithmetic<"qadd_predicated", "vqaddq", (? (unsignedflag Scalar)), 0>;
486 defm : VectorScalarArithmetic<"hadd_predicated", "vhaddq", (? (unsignedflag Scalar))>;
487 defm : VectorScalarArithmetic<"qsub_predicated", "vqsubq", (? (unsignedflag Scalar)), 0>;
488 defm : VectorScalarArithmetic<"hsub_predicated", "vhsubq", (? (unsignedflag Scalar))>;
490 let params = T.Signed in {
491 defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>;
492 defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>;
493 def vminaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred),
494 (IRInt<"vmina_predicated", [UVector,Predicate]> $a, $b, $pred)>;
495 def vmaxaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred),
496 (IRInt<"vmaxa_predicated", [UVector,Predicate]> $a, $b, $pred)>;
498 defm : VectorScalarArithmetic<"qdmulh_predicated", "vqdmulhq", (?), 0>;
499 defm : VectorScalarArithmetic<"qrdmulh_predicated", "vqrdmulhq", (?), 0>;
502 let params = T.Poly, overrideKindLetter = "p" in {
503 defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>;
504 defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>;
507 let params = [s16, s32] in {
508 def vqdmullbq: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
509 (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 0)>;
510 def vqdmulltq: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
511 (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 1)>;
512 defm vqdmullbq: DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 0)), 0>;
513 defm vqdmulltq: DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 1)), 0>;
515 let pnt = PNT_NType in {
516 def vqdmullbq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b),
517 (IRInt<"vqdmull", [DblVector, Vector]>
519 def vqdmulltq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b),
520 (IRInt<"vqdmull", [DblVector, Vector]>
523 defm vqdmullbq_n: DblVectorScalarArithmetic<"vqdmull_predicated",
524 "vqdmullbq", (? (u32 0)), 0>;
525 defm vqdmulltq_n: DblVectorScalarArithmetic<"vqdmull_predicated",
526 "vqdmulltq", (? (u32 1)), 0>;
529 // Predicated intrinsics - Float types only
530 let params = T.Float in {
531 defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>;
532 defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>;
533 def vminnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
534 (IRInt<"vminnma_predicated", [Vector,Predicate]> $a, $b, $pred)>;
535 def vmaxnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
536 (IRInt<"vmaxnma_predicated", [Vector,Predicate]> $a, $b, $pred)>;
539 multiclass Reduction<Type Accumulator, string basename, list<Type> basetypes,
541 dag postCG = (seq (id $ret)),
542 dag accArg = (args Accumulator:$prev),
544 defvar intArgsBase = (? $prev, $vec);
545 defvar intArgsUnpred = !con(intArgsBase,
546 !if(needSign, (? (unsignedflag Scalar)), (?)));
547 defvar intArgsPred = !con(intArgsUnpred, (? $pred));
548 defvar intUnpred = !setop(intArgsUnpred, IRInt<basename, basetypes>);
549 defvar intPred = !setop(intArgsPred, IRInt<
550 basename#"_predicated", !listconcat(basetypes, [Predicate])>);
553 Accumulator, !con(accArg, (args Vector:$vec)),
554 !con(preCG, (seq intUnpred:$ret), postCG)>;
556 Accumulator, !con(accArg, (args Vector:$vec, Predicate:$pred)),
557 !con(preCG, (seq intPred:$ret), postCG)>;
560 let params = T.Int in {
561 defm vminvq: Reduction<Scalar, "minv", [Vector], 1, (seq (Scalar $ret))>;
562 defm vmaxvq: Reduction<Scalar, "maxv", [Vector], 1, (seq (Scalar $ret))>;
565 let params = T.Signed in {
566 defm vminavq: Reduction<UScalar, "minav", [Vector], 0, (seq (UScalar $ret))>;
567 defm vmaxavq: Reduction<UScalar, "maxav", [Vector], 0, (seq (UScalar $ret))>;
570 let params = T.Float in {
571 defm vminnmvq: Reduction<Scalar, "minnmv", [Scalar, Vector]>;
572 defm vmaxnmvq: Reduction<Scalar, "maxnmv", [Scalar, Vector]>;
573 defm vminnmavq: Reduction<Scalar, "minnmav", [Scalar, Vector]>;
574 defm vmaxnmavq: Reduction<Scalar, "maxnmav", [Scalar, Vector]>;
577 foreach half = [ "b", "t" ] in {
578 defvar halfconst = !if(!eq(half, "b"), 0, 1);
580 let params = [f32], pnt = PNT_None in {
581 def vcvt#half#q_f16: Intrinsic<
582 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a),
583 (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>;
584 def vcvt#half#q_m_f16: Intrinsic<
585 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred),
586 (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>;
587 } // params = [f32], pnt = PNT_None
589 let params = [f16], pnt = PNT_None in {
590 def vcvt#half#q_f32: Intrinsic<VecOf<f32>, (args Vector:$a),
591 (IRInt<"vcvt_widen"> $a, halfconst)>;
592 defm vcvt#half#q: IntrinsicMX<
593 VecOf<f32>, (args Vector:$a, PredOf<f32>:$pred),
594 (IRInt<"vcvt_widen_predicated"> $inactive, $a, halfconst, $pred),
596 } // params = [f16], pnt = PNT_None
597 } // loop over half = "b", "t"
599 multiclass float_int_conversions<Type FScalar, Type IScalar, IRBuilderBase ftoi, IRBuilderBase itof> {
600 defvar FVector = VecOf<FScalar>;
601 defvar IVector = VecOf<IScalar>;
603 let params = [IScalar] in {
604 let pnt = PNT_2Type in {
605 def : Intrinsic<FVector, (args IVector:$a), (itof $a, FVector)>,
606 NameOverride<"vcvtq_" # FScalar>;
608 defm vcvtq: IntrinsicMX<FVector, (args IVector:$a, Predicate:$pred),
609 (IRInt<"vcvt_fp_int_predicated", [FVector, IVector, Predicate]>
610 $a, (unsignedflag IScalar), $pred, $inactive),
611 1, "_" # FScalar, PNT_2Type, PNT_2Type>;
613 let params = [FScalar] in {
614 let pnt = PNT_None in {
615 def : Intrinsic<IVector, (args FVector:$a), (ftoi $a, IVector)>,
616 NameOverride<"vcvtq_" # IScalar>;
618 foreach suffix = ["a","n","p","m"] in
619 def : Intrinsic<IVector, (args FVector:$a),
620 (IRInt<"vcvt"#suffix, [IVector, FVector]>
621 (unsignedflag IScalar), $a)>,
622 NameOverride<"vcvt"#suffix#"q_" # IScalar>;
624 defm vcvtq: IntrinsicMX<IVector, (args FVector:$a, Predicate:$pred),
625 (IRInt<"vcvt_fp_int_predicated", [IVector, FVector, Predicate]>
626 $a, (unsignedflag IScalar), $pred, $inactive),
627 1, "_" # IScalar, PNT_2Type, PNT_None>;
629 foreach suffix = ["a","n","p","m"] in {
630 defm "vcvt"#suffix#"q" : IntrinsicMX<
631 IVector, (args FVector:$a, Predicate:$pred),
632 (IRInt<"vcvt"#suffix#"_predicated", [IVector, FVector, Predicate]>
633 (unsignedflag IScalar), $inactive, $a, $pred),
634 1, "_" # IScalar, PNT_2Type, PNT_None>;
639 defm "" : float_int_conversions<f32, u32, fptoui, uitofp>;
640 defm "" : float_int_conversions<f16, u16, fptoui, uitofp>;
641 defm "" : float_int_conversions<f32, s32, fptosi, sitofp>;
642 defm "" : float_int_conversions<f16, s16, fptosi, sitofp>;
644 multiclass vmovl<bit top> {
645 let params = [s8, u8, s16, u16] in {
646 def "": Intrinsic<DblVector, (args Vector:$a),
647 (extend (unzip $a, top), DblVector, (unsignedflag Scalar))>;
648 defm "": IntrinsicMX<DblVector, (args Vector:$a, DblPredicate:$pred),
649 (IRInt<"vmovl_predicated", [DblVector, Vector, DblPredicate]>
650 $a, (unsignedflag Scalar), top, $pred, $inactive)>;
654 defm vmovlbq: vmovl<0>;
655 defm vmovltq: vmovl<1>;
657 multiclass vmovn<bit top, dag wide_result> {
658 let params = [s16, u16, s32, u32] in {
659 def "": Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a),
660 (trunc wide_result, HalfVector)>;
661 def _m: Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a,
663 (IRInt<"vmovn_predicated", [HalfVector, Vector, Predicate]>
664 $inactive, $a, top, $pred)>;
668 defm vmovntq: vmovn<1, (zip (vreinterpret $inactive, Vector), $a)>;
669 defm vmovnbq: vmovn<0,
670 (zip $a, (vreinterpret (vrev $inactive, (bitsize Scalar)), Vector))>;
672 multiclass vqmovn<bit top, Type RetScalar> {
673 defvar RetVector = VecOf<RetScalar>;
675 let params = [s16, u16, s32, u32] in {
677 RetVector, (args RetVector:$inactive, Vector:$a),
678 (IRInt<"vqmovn", [RetVector, Vector]>
679 $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), top)>,
682 RetVector, (args RetVector:$inactive, Vector:$a, Predicate:$pred),
683 (IRInt<"vqmovn_predicated", [RetVector, Vector, Predicate]>
684 $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar),
686 NameOverride<NAME # "_m">;
690 let params = [s16, s32, u16, u32] in {
691 defm vqmovntq: vqmovn<1, HalfScalar>;
692 defm vqmovnbq: vqmovn<0, HalfScalar>;
694 let params = [s16, s32] in {
695 defm vqmovuntq: vqmovn<1, UHalfScalar>;
696 defm vqmovunbq: vqmovn<0, UHalfScalar>;
699 multiclass vrnd<IRIntBase ir_int, string suffix> {
700 let params = T.Float in {
701 def "": Intrinsic<Vector, (args Vector:$a), (ir_int $a)>;
702 defm "": IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
703 (IRInt<"vrint"#suffix#"_predicated", [Vector, Predicate]>
704 $a, $pred, $inactive)>;
708 defm vrndq: vrnd<IRIntBase<"trunc", [Vector]>, "z">;
709 defm vrndmq: vrnd<IRIntBase<"floor", [Vector]>, "m">;
710 defm vrndpq: vrnd<IRIntBase<"ceil", [Vector]>, "p">;
711 defm vrndaq: vrnd<IRIntBase<"round", [Vector]>, "a">;
712 defm vrndxq: vrnd<IRIntBase<"rint", [Vector]>, "x">;
713 defm vrndnq: vrnd<IRInt<"vrintn", [Vector]>, "n">;
715 multiclass compare_with_pred<string condname, dag arguments,
716 dag cmp, string suffix> {
717 // Make the predicated and unpredicated versions of a single comparison.
718 def: Intrinsic<Predicate, arguments,
719 (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>,
720 NameOverride<"vcmp" # condname # "q" # suffix>;
721 def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)),
722 (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>,
723 NameOverride<"vcmp" # condname # "q_m" # suffix>;
726 multiclass compare<string condname, IRBuilder cmpop> {
727 // Make all four variants of a comparison: the vector/vector and
728 // vector/scalar forms, each using compare_with_pred to make a
729 // predicated and unpredicated version.
730 defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb),
731 (cmpop $va, $vb), "">;
732 let pnt = PNT_NType in {
733 defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb),
734 (cmpop $va, (splat $sb)), "_n">;
737 let params = T.Int in {
738 defm: compare<"eq", icmp_eq>;
739 defm: compare<"ne", icmp_ne>;
741 let params = T.Signed in {
742 defm: compare<"gt", icmp_sgt>;
743 defm: compare<"ge", icmp_sge>;
744 defm: compare<"lt", icmp_slt>;
745 defm: compare<"le", icmp_sle>;
747 let params = T.Unsigned in {
748 defm: compare<"hi", icmp_ugt>;
749 defm: compare<"cs", icmp_uge>;
751 let params = T.Float in {
752 defm: compare<"eq", fcmp_eq>;
753 defm: compare<"ne", fcmp_ne>;
754 defm: compare<"gt", fcmp_gt>;
755 defm: compare<"ge", fcmp_ge>;
756 defm: compare<"lt", fcmp_lt>;
757 defm: compare<"le", fcmp_le>;
760 let params = T.Signed in {
761 def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
762 (select (icmp_sle $a, $b), $a, $b)>;
763 def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
764 (select (icmp_sge $a, $b), $a, $b)>;
765 def vminaq: Intrinsic<UVector, (args UVector:$a, Vector:$b),
766 (seq (select (icmp_slt $b, (zeroinit Vector)),
767 (sub (zeroinit Vector), $b), $b):$absb,
768 (select (icmp_ule $a, $absb), $a, $absb))>;
769 def vmaxaq: Intrinsic<UVector, (args UVector:$a, Vector:$b),
770 (seq (select (icmp_slt $b, (zeroinit Vector)),
771 (sub (zeroinit Vector), $b), $b):$absb,
772 (select (icmp_uge $a, $absb), $a, $absb))>;
774 let params = T.Unsigned in {
775 def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
776 (select (icmp_ule $a, $b), $a, $b)>,
777 NameOverride<"vminq">;
778 def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
779 (select (icmp_uge $a, $b), $a, $b)>,
780 NameOverride<"vmaxq">;
782 let params = T.Float in {
783 def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
784 (IRIntBase<"minnum", [Vector]> $a, $b)>;
785 def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
786 (IRIntBase<"maxnum", [Vector]> $a, $b)>;
787 def vminnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
788 (IRIntBase<"minnum", [Vector]>
789 (IRIntBase<"fabs", [Vector]> $a),
790 (IRIntBase<"fabs", [Vector]> $b))>;
791 def vmaxnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
792 (IRIntBase<"maxnum", [Vector]>
793 (IRIntBase<"fabs", [Vector]> $a),
794 (IRIntBase<"fabs", [Vector]> $b))>;
797 def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred),
798 (select $pred, $t, $f)> { let params = T.Usual; }
799 def vpselq_64: Intrinsic<
800 Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred),
801 (bitcast (select $pred, (bitcast $t, VecOf<u32>),
802 (bitcast $f, VecOf<u32>)), Vector)>,
803 NameOverride<"vpselq"> { let params = T.All64; }
805 let params = [Void], pnt = PNT_None in {
807 multiclass vctp<Type pred, string intname> {
808 def "": Intrinsic<pred, (args u32:$val),
809 (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>;
810 def _m: Intrinsic<pred, (args u32:$val, pred:$inpred),
811 (u16 (IRInt<"pred_v2i", [pred]> (and $inpred,
812 (IRIntBase<intname> $val))))>;
814 defm vctp8q: vctp<PredOf<u8>, "arm_mve_vctp8">;
815 defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">;
816 defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">;
817 defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">;
819 def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred),
820 (xor $pred, (u16 65535))>;
824 multiclass contiguous_load<string mnemonic, PrimitiveType memtype,
825 list<Type> same_size, list<Type> wider> {
826 // Intrinsics named with explicit memory and element sizes that match:
827 // vldrbq_?8, vldrhq_?16, vldrwq_?32.
828 let params = same_size, pnt = PNT_None in {
829 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
830 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
831 NameOverride<mnemonic>;
832 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
834 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
835 (CPtr<Vector> $addr), !srl(memtype.size,3),
836 $pred, (zeroinit Vector))>,
837 NameOverride<mnemonic # "_z">;
840 // Synonyms for the above, with the generic name vld1q that just means
841 // 'memory and element sizes match', and allows convenient polymorphism with
842 // the memory and element types covariant.
843 let params = same_size in {
844 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
845 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
846 NameOverride<"vld1q">;
847 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
849 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
850 (CPtr<Vector> $addr), !srl(memtype.size,3),
851 $pred, (zeroinit Vector))>,
852 NameOverride<"vld1q_z">;
855 // Intrinsics with the memory size narrower than the vector element, so that
856 // they load less than 128 bits of memory and sign/zero extend each loaded
857 // value into a wider vector lane.
858 let params = wider, pnt = PNT_None in {
859 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
860 (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>>
861 $addr), !srl(memtype.size,3))),
862 Vector, (unsignedflag Scalar))>,
863 NameOverride<mnemonic>;
864 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
866 (extend (IRIntBase<"masked_load",
867 [NarrowedVecOf<memtype,Vector>,
868 CPtr<NarrowedVecOf<memtype,Vector>>]>
869 (CPtr<NarrowedVecOf<memtype,Vector>> $addr),
870 !srl(memtype.size,3), $pred,
871 (zeroinit NarrowedVecOf<memtype,Vector>)),
872 Vector, (unsignedflag Scalar))>,
873 NameOverride<mnemonic # "_z">;
877 defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
878 defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>;
879 defm: contiguous_load<"vldrwq", u32, T.All32, []>;
881 multiclass contiguous_store<string mnemonic, PrimitiveType memtype,
882 list<Type> same_size, list<Type> wider> {
883 // Intrinsics named with explicit memory and element sizes that match:
884 // vstrbq_?8, vstrhq_?16, vstrwq_?32.
885 let params = same_size in {
886 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
889 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
890 NameOverride<mnemonic>;
891 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
892 Vector:$value, Predicate:$pred),
893 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
894 $value, (Ptr<Vector> $addr),
895 !srl(memtype.size,3), $pred)>,
896 NameOverride<mnemonic # "_p">;
899 // Synonyms for the above, with the generic name vst1q that just means
900 // 'memory and element sizes match', and allows convenient polymorphism with
901 // the memory and element types covariant.
902 let params = same_size in {
903 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
906 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
907 NameOverride<"vst1q">;
908 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
909 Vector:$value, Predicate:$pred),
910 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
911 $value, (Ptr<Vector> $addr),
912 !srl(memtype.size,3), $pred)>,
913 NameOverride<"vst1q_p">;
916 // Intrinsics with the memory size narrower than the vector element, so that
917 // they store less than 128 bits of memory, truncating each vector lane into
918 // a narrower value to store.
919 let params = wider in {
920 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
922 (store (trunc $value, NarrowedVecOf<memtype,Vector>),
923 (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
924 !srl(memtype.size,3)))>,
925 NameOverride<mnemonic>;
926 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
927 Vector:$value, Predicate:$pred),
928 (IRIntBase<"masked_store",
929 [NarrowedVecOf<memtype,Vector>,
930 Ptr<NarrowedVecOf<memtype,Vector>>]>
931 (trunc $value, NarrowedVecOf<memtype,Vector>),
932 (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
933 !srl(memtype.size,3), $pred)>,
934 NameOverride<mnemonic # "_p">;
938 defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
939 defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>;
940 defm: contiguous_store<"vstrwq", u32, T.All32, []>;
942 multiclass gather_base<list<Type> types, int size> {
943 let params = types, pnt = PNT_None in {
944 def _gather_base: Intrinsic<
945 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset),
946 (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>;
948 def _gather_base_z: Intrinsic<
949 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred),
950 (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]>
951 $addr, $offset, $pred)>;
953 def _gather_base_wb: Intrinsic<
954 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset),
955 (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]>
956 (load $addr), $offset):$pair,
957 (store (xval $pair, 1), $addr),
960 def _gather_base_wb_z: Intrinsic<
961 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
963 (seq (IRInt<"vldr_gather_base_wb_predicated",
964 [Vector, UVector, Predicate]>
965 (load $addr), $offset, $pred):$pair,
966 (store (xval $pair, 1), $addr),
971 defm vldrwq: gather_base<T.All32, 4>;
972 defm vldrdq: gather_base<T.All64, 8>;
974 multiclass scatter_base<list<Type> types, int size> {
975 let params = types in {
976 def _scatter_base: Intrinsic<
977 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data),
978 (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>;
980 def _scatter_base_p: Intrinsic<
981 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data,
983 (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]>
984 $addr, $offset, $data, $pred)>;
986 def _scatter_base_wb: Intrinsic<
987 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data),
988 (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]>
989 (load $addr), $offset, $data):$wbaddr,
990 (store $wbaddr, $addr))>;
992 def _scatter_base_wb_p: Intrinsic<
993 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
994 Vector:$data, Predicate:$pred),
995 (seq (IRInt<"vstr_scatter_base_wb_predicated",
996 [UVector, Vector, Predicate]>
997 (load $addr), $offset, $data, $pred):$wbaddr,
998 (store $wbaddr, $addr))>;
1002 defm vstrwq: scatter_base<T.All32, 4>;
1003 defm vstrdq: scatter_base<T.All64, 8>;
1005 multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> {
1006 let params = types in {
1007 def _gather_offset: Intrinsic<
1008 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
1009 (IRInt<"vldr_gather_offset",
1010 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
1011 $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>;
1012 def _gather_offset_z: Intrinsic<
1013 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1015 (IRInt<"vldr_gather_offset_predicated",
1016 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
1017 $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>;
1021 multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype,
1023 let params = types in {
1024 def _gather_shifted_offset: Intrinsic<
1025 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
1026 (IRInt<"vldr_gather_offset",
1027 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
1028 $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>;
1029 def _gather_shifted_offset_z: Intrinsic<
1030 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1032 (IRInt<"vldr_gather_offset_predicated",
1033 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
1034 $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>;
1038 multiclass gather_offset_both<list<Type> types, PrimitiveType memtype,
1040 defm "": gather_offset_unshifted<types, memtype>;
1041 defm "": gather_offset_shifted<types, memtype, shift>;
1044 defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>;
1045 defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
1046 defm vldrwq: gather_offset_both<T.All32, u32, 2>;
1047 defm vldrdq: gather_offset_both<T.Int64, u64, 3>;
1049 multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> {
1050 let params = types in {
1051 def _scatter_offset: Intrinsic<
1052 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1054 (IRInt<"vstr_scatter_offset",
1055 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
1056 $base, $offsets, $data, memtype.size, 0)>;
1057 def _scatter_offset_p: Intrinsic<
1058 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1059 Vector:$data, Predicate:$pred),
1060 (IRInt<"vstr_scatter_offset_predicated",
1061 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
1062 $base, $offsets, $data, memtype.size, 0, $pred)>;
1066 multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype,
1068 let params = types in {
1069 def _scatter_shifted_offset: Intrinsic<
1070 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1072 (IRInt<"vstr_scatter_offset",
1073 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
1074 $base, $offsets, $data, memtype.size, shift)>;
1075 def _scatter_shifted_offset_p: Intrinsic<
1076 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1077 Vector:$data, Predicate:$pred),
1078 (IRInt<"vstr_scatter_offset_predicated",
1079 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
1080 $base, $offsets, $data, memtype.size, shift, $pred)>;
1084 multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype,
1086 defm "": scatter_offset_unshifted<types, memtype>;
1087 defm "": scatter_offset_shifted<types, memtype, shift>;
1090 defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>;
1091 defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
1092 defm vstrwq: scatter_offset_both<T.All32, u32, 2>;
1093 defm vstrdq: scatter_offset_both<T.Int64, u64, 3>;
1095 let params = T.Int in {
1096 def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
1097 (shl $v, (splat (Scalar $sh)))>;
1098 defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh,
1100 (IRInt<"shl_imm_predicated", [Vector, Predicate]>
1101 $v, $sh, $pred, $inactive), 1, "_n">;
1103 let pnt = PNT_NType in {
1104 def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
1105 (immshr $v, $sh, (unsignedflag Scalar))>;
1106 defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
1108 (IRInt<"shr_imm_predicated", [Vector, Predicate]>
1109 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
1113 let params = T.Int in {
1114 def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
1115 (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
1116 def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v,
1117 imm_0toNm1:$sh, Predicate:$pred),
1118 (IRInt<"vqshl_imm_predicated", [Vector, Predicate]>
1119 $v, $sh, (unsignedflag Scalar), $pred, $inactive)>;
1121 let pnt = PNT_NType in {
1122 def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
1123 (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
1124 defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
1126 (IRInt<"vrshr_imm_predicated", [Vector, Predicate]>
1127 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
1131 let params = T.Signed, pnt = PNT_NType in {
1132 def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh),
1133 (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>;
1134 def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v,
1135 imm_0toNm1:$sh, Predicate:$pred),
1136 (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]>
1137 $v, $sh, $pred, $inactive)>;
1140 multiclass vshll_imm<int top> {
1141 let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in {
1142 def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh),
1143 (IRInt<"vshll_imm", [DblVector, Vector]>
1144 $v, $sh, (unsignedflag Scalar), top)>;
1145 defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh,
1146 DblPredicate:$pred),
1147 (IRInt<"vshll_imm_predicated", [DblVector, Vector, DblPredicate]>
1148 $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">;
1151 defm vshllbq : vshll_imm<0>;
1152 defm vshlltq : vshll_imm<1>;
1154 multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME,
1155 dag extraargs = (?)> {
1156 defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)),
1157 [Vector], [outtype, Vector]);
1160 outtype, (args outtype:$a, Vector:$b, imm:$sh),
1161 !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>;
1163 def q_m_n: Intrinsic<
1164 outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred),
1165 !con((IRInt<intname # "_predicated", intparams # [Predicate]>
1166 $a, $b, $sh), extraargs, (? $pred))>;
1169 multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> {
1170 defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>;
1171 defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>;
1174 let params = [s16, s32, u16, u32], pnt = PNT_NType in {
1175 defvar U = (unsignedflag Scalar);
1176 defm vshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>;
1177 defm vqshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>;
1178 defm vrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>;
1179 defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>;
1181 let params = [s16, s32], pnt = PNT_NType in {
1182 defm vqshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>;
1183 defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>;
1185 let params = T.Int, pnt = PNT_NType in {
1186 defm vsli : DyadicImmShift<Vector, imm_0toNm1>;
1187 defm vsri : DyadicImmShift<Vector, imm_1toN>;
1190 multiclass VSHL_non_imm<string scalarSuffix, int q, int r,
1191 PolymorphicNameType pnt_scalar_unpred = PNT_Type> {
1192 let pnt = pnt_scalar_unpred in {
1193 def scalarSuffix: Intrinsic<
1194 Vector, (args Vector:$in, s32:$sh),
1195 (IRInt<"vshl_scalar", [Vector]> $in, $sh,
1196 q, r, (unsignedflag Scalar))>;
1198 def "_m" # scalarSuffix: Intrinsic<
1199 Vector, (args Vector:$in, s32:$sh, Predicate:$pred),
1200 (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh,
1201 q, r, (unsignedflag Scalar), $pred)>;
1204 Vector, (args Vector:$in, SVector:$sh),
1205 (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh,
1206 q, r, (unsignedflag Scalar))>;
1207 defm "": IntrinsicMX<
1208 Vector, (args Vector:$in, SVector:$sh, Predicate:$pred),
1209 (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh,
1210 q, r, (unsignedflag Scalar), $pred, $inactive),
1211 // The saturating shift intrinsics don't have an x variant, so we
1212 // set wantXVariant to 1 iff q == 0
1216 let params = T.Int in {
1217 defm vshlq : VSHL_non_imm<"_r", 0, 0>;
1218 defm vqshlq : VSHL_non_imm<"_r", 1, 0>;
1219 defm vrshlq : VSHL_non_imm<"_n", 0, 1, PNT_NType>;
1220 defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>;
1223 // Base class for the scalar shift intrinsics.
1224 class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
1225 Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> {
1226 let params = [Void];
1230 // Subclass that includes the machinery to take a 64-bit input apart
1231 // into halves, retrieve the two halves of a shifted output as a pair,
1232 // and glue the pieces of the pair back into an i64 for output.
1233 class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
1234 ScalarShift<argtype, shiftCountArg,
1235 (seq (u32 (lshr $value, (argtype 32))):$hi,
1238 (or (shl (u64 (xval $pair, 1)), (u64 32)),
1239 (u64 (xval $pair, 0))))>;
1241 // The family of saturating/rounding scalar shifts that take an
1242 // immediate shift count. They come in matched 32- and 64-bit pairs.
1243 multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> {
1244 def "": ScalarShift<arg32, (args imm_1to32:$sh),
1245 (IRInt<NAME> $value, $sh)>;
1246 def l: LongScalarShift<arg64, (args imm_1to32:$sh),
1247 (IRInt<NAME # "l"> $lo, $hi, $sh)>;
1249 defm uqshl: ScalarSaturatingShiftImm<u32, u64>;
1250 defm urshr: ScalarSaturatingShiftImm<u32, u64>;
1251 defm sqshl: ScalarSaturatingShiftImm<s32, s64>;
1252 defm srshr: ScalarSaturatingShiftImm<s32, s64>;
1254 // The family of saturating/rounding scalar shifts that take a
1255 // register shift count. They also have 32- and 64-bit forms, but the
1256 // 64-bit form also has a version that saturates to 48 bits, so the IR
1257 // intrinsic takes an extra saturation-type operand.
1258 multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> {
1259 def "": ScalarShift<arg32, (args s32:$sh),
1260 (IRInt<NAME> $value, $sh)>;
1261 def l: LongScalarShift<arg64, (args s32:$sh),
1262 (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>;
1263 def l_sat48: LongScalarShift<arg64, (args s32:$sh),
1264 (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>;
1266 defm uqrshl: ScalarSaturatingShiftReg<u32, u64>;
1267 defm sqrshr: ScalarSaturatingShiftReg<s32, s64>;
1269 // The intrinsics for LSLL and ASRL come in 64-bit versions only, with
1270 // no saturation count.
1271 def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>;
1272 def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>;
1274 multiclass vadcsbc {
1275 def q: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
1276 (seq (IRInt<NAME, [Vector]> $a, $b, (shl (load $carry), 29)):$pair,
1277 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1279 def iq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
1280 (seq (IRInt<NAME, [Vector]> $a, $b, 0):$pair,
1281 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1283 def q_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
1284 Ptr<uint>:$carry, Predicate:$pred),
1285 (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
1286 (shl (load $carry), 29), $pred):$pair,
1287 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1289 def iq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
1290 Ptr<uint>:$carry, Predicate:$pred),
1291 (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
1293 (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1296 let params = T.Int32 in {
1301 let params = T.Int in {
1302 def vshlcq: Intrinsic<
1303 Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm),
1305 (IRInt<"vshlc", [Vector]> $v, $s, $imm):$pair,
1306 (store (xval $pair, 0), $ps),
1308 def vshlcq_m: Intrinsic<
1309 Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm, Predicate:$pred),
1311 (IRInt<"vshlc_predicated", [Vector, Predicate]>
1312 $v, $s, $imm, $pred):$pair,
1313 (store (xval $pair, 0), $ps),
1317 multiclass VectorComplexAddPred<dag not_halving, dag angle> {
1318 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
1319 (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>;
1320 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
1321 (IRInt<"vcaddq_predicated", [Vector, Predicate]>
1322 not_halving, angle, $inactive, $a, $b, $pred)>;
1325 multiclass VectorComplexMulPred<dag angle> {
1326 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
1327 (IRInt<"vcmulq", [Vector]> angle, $a, $b)>;
1328 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
1329 (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b,
1333 multiclass VectorComplexMLAPred<dag angle> {
1334 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
1335 (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>;
1336 def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
1338 (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>;
1341 multiclass VectorComplexAddAngle<dag not_halving> {
1342 defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>;
1343 defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>;
1346 multiclass VectorComplexMulAngle {
1347 defm "" : VectorComplexMulPred<(u32 0)>;
1348 defm _rot90 : VectorComplexMulPred<(u32 1)>;
1349 defm _rot180 : VectorComplexMulPred<(u32 2)>;
1350 defm _rot270 : VectorComplexMulPred<(u32 3)>;
1353 multiclass VectorComplexMLAAngle {
1354 defm "" : VectorComplexMLAPred<(u32 0)>;
1355 defm _rot90 : VectorComplexMLAPred<(u32 1)>;
1356 defm _rot180 : VectorComplexMLAPred<(u32 2)>;
1357 defm _rot270 : VectorComplexMLAPred<(u32 3)>;
1360 let params = T.Usual in
1361 defm vcaddq : VectorComplexAddAngle<(u32 1)>;
1363 let params = T.Signed in
1364 defm vhcaddq : VectorComplexAddAngle<(u32 0)>;
1366 let params = T.Float in {
1367 defm vcmulq : VectorComplexMulAngle;
1368 defm vcmlaq : VectorComplexMLAAngle;
1371 multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> {
1373 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b),
1374 (IRInt<"vmldava", [Vector]>
1375 (unsignedflag Scalar), subtract, exchange,
1376 (zeroinit Scalar32), $a, $b)>;
1378 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred),
1379 (IRInt<"vmldava_predicated", [Vector, Predicate]>
1380 (unsignedflag Scalar), subtract, exchange,
1381 (zeroinit Scalar32), $a, $b, $pred)>;
1384 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c),
1385 (IRInt<"vmldava", [Vector]>
1386 (unsignedflag Scalar), subtract, exchange,
1388 def "a"#xsuffix#"q_p"
1389 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c,
1391 (IRInt<"vmldava_predicated", [Vector, Predicate]>
1392 (unsignedflag Scalar), subtract, exchange,
1393 $a, $b, $c, $pred)>;
1396 class IntrSplit64<Type resty, dag args, dag codegen>
1397 : Intrinsic<resty, args,
1398 (seq (u32 (lshr $a, (u64 32))):$hi,
1401 (or (shl (u64 (xval $pair, 1)), (u64 32)),
1402 (u64 (xval $pair, 0))))>;
1404 class IntrSplit64ZeroInit<Type resty, dag args, dag codegen>
1405 : Intrinsic<resty, args,
1406 (seq (zeroinit u32):$hi,
1409 (or (shl (u64 (xval $pair, 1)), (u64 32)),
1410 (u64 (xval $pair, 0))))>;
1412 multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange,
1413 string xsuffix, string irname> {
1415 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b),
1416 (IRInt<irname, [Vector]>
1417 (unsignedflag Scalar), subtract, exchange,
1420 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b,
1422 (IRInt<irname#"_predicated", [Vector, Predicate]>
1423 (unsignedflag Scalar), subtract, exchange,
1424 $lo, $hi, $a, $b, $pred)>;
1427 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c),
1428 (IRInt<irname, [Vector]>
1429 (unsignedflag Scalar), subtract, exchange,
1431 def "a"#xsuffix#"q_p"
1432 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c,
1434 (IRInt<irname#"_predicated", [Vector, Predicate]>
1435 (unsignedflag Scalar), subtract, exchange,
1436 $lo, $hi, $b, $c, $pred)>;
1439 multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> {
1440 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">;
1443 multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> {
1444 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix,
1448 multiclass VADDV<bit acc, bit pred, string intbase, Type Scalar> {
1449 defvar accArg = !if(acc, (args Scalar:$acc), (args));
1450 defvar predArg = !if(pred, (args Predicate:$pred), (args));
1451 defvar intrinsic = !if(pred,
1452 IRInt<intbase # "_predicated", [Vector, Predicate]>,
1453 IRInt<intbase, [Vector]>);
1454 defvar intCG = !con((intrinsic $v, (unsignedflag Scalar)),
1455 !if(pred, (? $pred), (?)));
1456 defvar accCG = !if(acc, (add intCG, $acc), intCG);
1458 def "": Intrinsic<Scalar, !con(accArg, (args Vector:$v), predArg), accCG>;
1461 let params = T.Int in {
1462 defm vaddvq : VADDV<0, 0, "addv", Scalar32>;
1463 defm vaddvaq : VADDV<1, 0, "addv", Scalar32>;
1464 defm vaddvq_p : VADDV<0, 1, "addv", Scalar32>;
1465 defm vaddvaq_p : VADDV<1, 1, "addv", Scalar32>;
1468 let params = [s32, u32] in {
1469 defm vaddlvq : VADDV<0, 0, "addlv", Scalar64>;
1470 defm vaddlvaq : VADDV<1, 0, "addlv", Scalar64>;
1471 defm vaddlvq_p : VADDV<0, 1, "addlv", Scalar64>;
1472 defm vaddlvaq_p : VADDV<1, 1, "addlv", Scalar64>;
1475 let params = T.Int in {
1476 def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c),
1477 (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>;
1478 def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c,
1480 (IRInt<"vabav_predicated", [Vector, Predicate]>
1481 (unsignedflag Scalar), $a, $b, $c, $pred)>;
1483 defm vmladav : MVEBinaryVectorHoriz32<V.False, V.False, "">;
1486 let params = T.Signed in {
1487 defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">;
1488 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">;
1489 defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">;
1492 let params = [u16, s16, u32, s32] in
1493 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">;
1495 let params = [s16, s32] in {
1496 defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">;
1497 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">;
1498 defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">;
1501 let params = T.Int32 in
1502 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">;
1504 let params = [s32] in {
1505 defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">;
1506 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">;
1507 defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">;
1510 multiclass vrev_predicated<int revsize> {
1511 defm "" : IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
1512 (IRInt<"vrev_predicated", [Vector, Predicate]>
1513 $a, revsize, $pred, $inactive)>;
1516 let params = T.All8 in {
1517 def vrev16q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 16)>;
1518 defm vrev16q: vrev_predicated<16>;
1520 let params = !listconcat(T.All8, T.All16) in {
1521 def vrev32q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 32)>;
1522 defm vrev32q: vrev_predicated<32>;
1524 let params = T.Usual in {
1525 def vrev64q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 64)>;
1526 defm vrev64q: vrev_predicated<64>;
1529 foreach desttype = T.All in {
1530 // We want a vreinterpretq between every pair of supported vector types
1531 // _except_ that there shouldn't be one from a type to itself.
1533 // So this foldl expression implements what you'd write in Python as
1534 // [srctype for srctype in T.All if srctype != desttype]
1535 let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist,
1536 !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype])))
1538 def "vreinterpretq_" # desttype: Intrinsic<
1539 VecOf<desttype>, (args Vector:$x), (vreinterpret $x, VecOf<desttype>)>;
1543 let params = T.All in {
1544 let pnt = PNT_None in {
1545 def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b),
1546 (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0),
1548 def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>;
1551 // This is the polymorphic form of vuninitializedq, which takes no type
1552 // suffix, but takes an _unevaluated_ vector parameter and returns an
1553 // uninitialized vector of the same vector type.
1555 // This intrinsic has no _non_-polymorphic form exposed to the user. But each
1556 // separately typed version of it still has to have its own clang builtin id,
1557 // which can't be called vuninitializedq_u32 or similar because that would
1558 // collide with the explicit nullary versions above. So I'm calling them
1559 // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that
1560 // full name never appears in the header file due to the polymorphicOnly
1561 // flag, and the _polymorphic suffix is omitted from the shortened name by
1562 // the custom PolymorphicNameType here.
1563 let polymorphicOnly = 1, nonEvaluating = 1,
1564 pnt = PolymorphicNameType<1, "polymorphic"> in {
1565 def vuninitializedq_polymorphic: Intrinsic<
1566 Vector, (args Vector), (undef Vector)>;
1569 def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane),
1570 (xelt_var $v, $lane)>;
1571 def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane),
1572 (ielt_var $v, $e, $lane)>;
1575 foreach desttype = !listconcat(T.Int16, T.Int32, T.Float) in {
1576 defvar is_dest_float = !eq(desttype.kind, "f");
1577 defvar is_dest_unsigned = !eq(desttype.kind, "u");
1578 // First immediate operand of the LLVM intrinsic
1579 defvar unsigned_flag = !if(is_dest_float, (unsignedflag Scalar),
1580 !if(is_dest_unsigned, V.True, V.False));
1581 // For float->int conversions _n and _x_n intrinsics are not polymorphic
1582 // because the signedness of the destination type cannot be inferred.
1583 defvar pnt_nx = !if(is_dest_float, PNT_2Type, PNT_None);
1585 let params = !if(is_dest_float,
1586 !if(!eq(desttype.size, 16), T.Int16, T.Int32),
1587 !if(!eq(desttype.size, 16), [f16], [f32])) in {
1589 def "vcvtq_n_"#desttype : Intrinsic<VecOf<desttype>,
1590 (args Vector:$a, imm_1toN:$b),
1591 (IRInt<"vcvt_fix", [VecOf<desttype>, Vector]> unsigned_flag, $a, $b)>;
1593 defm "vcvtq" : IntrinsicMX<VecOf<desttype>,
1594 (args Vector:$a, imm_1toN:$b, Predicate:$p),
1595 (IRInt<"vcvt_fix_predicated", [VecOf<desttype>, Vector, Predicate]>
1596 unsigned_flag, $inactive, $a, $b, $p),
1597 1, "_n_"#desttype, PNT_2Type, pnt_nx>;
1601 let params = T.Usual in {
1602 let pnt = PNT_NType in
1603 def vbrsrq_n: Intrinsic<Vector, (args Vector:$a, s32:$b),
1604 (IRInt<"vbrsr", [Vector]> $a, $b)>;
1605 defm vbrsrq : IntrinsicMX<Vector, (args Vector:$a, s32:$b, Predicate:$pred),
1606 (IRInt<"vbrsr_predicated", [Vector, Predicate]>
1607 $inactive, $a, $b, $pred), 1, "_n",
1608 PNT_NType, PNT_NType>;