1 //===--- arm_neon.td - ARM NEON compiler interface ------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the TableGen definitions from which the ARM NEON header
11 // file will be generated. See ARM document DUI0348B.
13 //===----------------------------------------------------------------------===//
15 // Each intrinsic is a subclass of the Inst class. An intrinsic can either
16 // generate a __builtin_* call or it can expand to a set of generic operations.
18 // The operations are subclasses of Operation providing a list of DAGs, the
19 // last of which is the return value. The available DAG nodes are documented
22 //===----------------------------------------------------------------------===//
24 // The base Operation class. All operations must subclass this.
25 class Operation<list<dag> ops=[]> {
29 // An operation that only contains a single DAG.
30 class Op<dag op> : Operation<[op]>;
31 // A shorter version of Operation - takes a list of DAGs. The last of these will
32 // be the return value.
33 class LOp<list<dag> ops> : Operation<ops>;
35 // These defs and classes are used internally to implement the SetTheory
36 // expansion and should be ignored.
37 foreach Index = 0-63 in
41 //===----------------------------------------------------------------------===//
42 // Available operations
43 //===----------------------------------------------------------------------===//
45 // DAG arguments can either be operations (documented below) or variables.
46 // Variables are prefixed with '$'. There are variables for each input argument,
47 // with the name $pN, where N starts at zero. So the zero'th argument will be
48 // $p0, the first $p1 etc.
50 // op - Binary or unary operator, depending on the number of arguments. The
51 // operator itself is just treated as a raw string and is not checked.
52 // example: (op "+", $p0, $p1) -> "__p0 + __p1".
53 // (op "-", $p0) -> "-__p0"
55 // call - Invoke another intrinsic. The input types are type checked and
56 // disambiguated. If there is no intrinsic defined that takes
57 // the given types (or if there is a type ambiguity) an error is
58 // generated at tblgen time. The name of the intrinsic is the raw
59 // name as given to the Inst class (not mangled).
60 // example: (call "vget_high", $p0) -> "vgetq_high_s16(__p0)"
61 // (assuming $p0 has type int16x8_t).
63 // cast - Perform a cast to a different type. This gets emitted as a static
64 // C-style cast. For a pure reinterpret cast (T x = *(T*)&y), use
67 // The syntax is (cast MOD* VAL). The last argument is the value to
68 // cast, preceded by a sequence of type modifiers. The target type
69 // starts off as the type of VAL, and is modified by MOD in sequence.
70 // The available modifiers are:
71 // - $X - Take the type of parameter/variable X. For example:
72 // (cast $p0, $p1) would cast $p1 to the type of $p0.
73 // - "R" - The type of the return type.
74 // - A typedef string - A NEON or stdint.h type that is then parsed.
75 // for example: (cast "uint32x4_t", $p0).
76 // - "U" - Make the type unsigned.
77 // - "S" - Make the type signed.
78 // - "H" - Halve the number of lanes in the type.
79 // - "D" - Double the number of lanes in the type.
80 // - "8" - Convert type to an equivalent vector of 8-bit signed
82 // example: (cast "R", "U", $p0) -> "(uint32x4_t)__p0" (assuming the return
83 // value is of type "int32x4_t".
84 // (cast $p0, "D", "8", $p1) -> "(int8x16_t)__p1" (assuming __p0
85 // has type float64x1_t or any other vector type of 64 bits).
86 // (cast "int32_t", $p2) -> "(int32_t)__p2"
88 // bitcast - Same as "cast", except a reinterpret-cast is produced:
89 // (bitcast "T", $p0) -> "*(T*)&__p0".
90 // The VAL argument is saved to a temporary so it can be used
93 // dup - Take a scalar argument and create a vector by duplicating it into
94 // all lanes. The type of the vector is the base type of the intrinsic.
95 // example: (dup $p1) -> "(uint32x2_t) {__p1, __p1}" (assuming the base type
98 // splat - Take a vector and a lane index, and return a vector of the same type
99 // containing repeated instances of the source vector at the lane index.
100 // example: (splat $p0, $p1) ->
101 // "__builtin_shufflevector(__p0, __p0, __p1, __p1, __p1, __p1)"
102 // (assuming __p0 has four elements).
104 // save_temp - Create a temporary (local) variable. The variable takes a name
105 // based on the zero'th parameter and can be referenced using
106 // using that name in subsequent DAGs in the same
107 // operation. The scope of a temp is the operation. If a variable
108 // with the given name already exists, an error will be given at
110 // example: [(save_temp $var, (call "foo", $p0)),
111 // (op "+", $var, $p1)] ->
112 // "int32x2_t __var = foo(__p0); return __var + __p1;"
114 // name_replace - Return the name of the current intrinsic with the first
115 // argument replaced by the second argument. Raises an error if
116 // the first argument does not exist in the intrinsic name.
117 // example: (call (name_replace "_high_", "_"), $p0) (to call the non-high
118 // version of this intrinsic).
120 // literal - Create a literal piece of code. The code is treated as a raw
121 // string, and must be given a type. The type is a stdint.h or
122 // NEON intrinsic type as given to (cast).
123 // example: (literal "int32_t", "0")
125 // shuffle - Create a vector shuffle. The syntax is (shuffle ARG0, ARG1, MASK).
126 // The MASK argument is a set of elements. The elements are generated
127 // from the two special defs "mask0" and "mask1". "mask0" expands to
128 // the lane indices in sequence for ARG0, and "mask1" expands to
129 // the lane indices in sequence for ARG1. They can be used as-is, e.g.
131 // (shuffle $p0, $p1, mask0) -> $p0
132 // (shuffle $p0, $p1, mask1) -> $p1
134 // or, more usefully, they can be manipulated using the SetTheory
135 // operators plus some extra operators defined in the NEON emitter.
136 // The operators are described below.
137 // example: (shuffle $p0, $p1, (add (highhalf mask0), (highhalf mask1))) ->
138 // A concatenation of the high halves of the input vectors.
141 // add, interleave, decimate: These set operators are vanilla SetTheory
142 // operators and take their normal definition.
146 // rotl - Rotate set left by a number of elements.
147 // example: (rotl mask0, 3) -> [3, 4, 5, 6, 0, 1, 2]
149 // rotl - Rotate set right by a number of elements.
150 // example: (rotr mask0, 3) -> [4, 5, 6, 0, 1, 2, 3]
152 // highhalf - Take only the high half of the input.
153 // example: (highhalf mask0) -> [4, 5, 6, 7] (assuming mask0 had 8 elements)
155 // highhalf - Take only the low half of the input.
156 // example: (lowhalf mask0) -> [0, 1, 2, 3] (assuming mask0 had 8 elements)
158 // rev - Perform a variable-width reversal of the elements. The zero'th argument
159 // is a width in bits to reverse. The lanes this maps to is determined
160 // based on the element width of the underlying type.
161 // example: (rev 32, mask0) -> [3, 2, 1, 0, 7, 6, 5, 4] (if 8-bit elements)
162 // example: (rev 32, mask0) -> [1, 0, 3, 2] (if 16-bit elements)
164 // mask0 - The initial sequence of lanes for shuffle ARG0
165 def mask0 : MaskExpand;
166 // mask0 - The initial sequence of lanes for shuffle ARG1
167 def mask1 : MaskExpand;
169 def OP_NONE : Operation;
170 def OP_UNAVAILABLE : Operation {
174 //===----------------------------------------------------------------------===//
175 // Instruction definitions
176 //===----------------------------------------------------------------------===//
178 // Every intrinsic subclasses "Inst". An intrinsic has a name, a prototype and
179 // a sequence of typespecs.
181 // The name is the base name of the intrinsic, for example "vget_lane". This is
182 // then mangled by the tblgen backend to add type information ("vget_lane_s16").
184 // A typespec is a sequence of uppercase characters (modifiers) followed by one
185 // lowercase character. A typespec encodes a particular "base type" of the
188 // An example typespec is "Qs" - quad-size short - uint16x8_t. The available
189 // typespec codes are given below.
191 // The string given to an Inst class is a sequence of typespecs. The intrinsic
192 // is instantiated for every typespec in the sequence. For example "sdQsQd".
194 // The prototype is a string that defines the return type of the intrinsic
195 // and the type of each argument. The return type and every argument gets a
196 // "modifier" that can change in some way the "base type" of the intrinsic.
198 // The modifier 'd' means "default" and does not modify the base type in any
199 // way. The available modifiers are given below.
212 // Typespec modifiers
213 // ------------------
214 // S: scalar, only used for function mangling.
217 // H: 128b without mangling 'q'
220 // Prototype modifiers
221 // -------------------
222 // prototype: return (arg, arg, ...)
225 // t: best-fit integer (int/poly args)
226 // x: signed integer (int/float args)
227 // u: unsigned integer (int/float args)
228 // f: float (int args)
229 // F: double (int args)
230 // H: half (int args)
232 // g: default, ignore 'Q' size modifier.
233 // j: default, force 'Q' size modifier.
234 // w: double width elements, same num elts
235 // n: double width elements, half num elts
236 // h: half width elements, double num elts
237 // q: half width elements, quad num elts
238 // e: half width elements, double num elts, unsigned
239 // m: half width elements, same num elts
241 // l: constant uint64
242 // s: scalar of element type
243 // z: scalar of half width element type, signed
244 // r: scalar of double width element type, signed
245 // a: scalar of element type (splat to vector type)
246 // b: scalar of unsigned integer/long type (int/float args)
247 // $: scalar of signed integer/long type (int/float args)
248 // y: scalar of float
249 // o: scalar of double
250 // k: default elt width, double num elts
251 // 2,3,4: array of default vectors
252 // B,C,D: array of default elts, force 'Q' size modifier.
254 // c: const pointer type
256 // Every intrinsic subclasses Inst.
257 class Inst <string n, string p, string t, Operation o> {
259 string Prototype = p;
261 string ArchGuard = "";
263 Operation Operation = o;
264 bit CartesianProductOfTypes = 0;
265 bit BigEndianSafe = 0;
267 bit isScalarShift = 0;
268 bit isScalarNarrowShift = 0;
270 // For immediate checks: the immediate will be assumed to specify the lane of
271 // a Q register. Only used for intrinsics which end up calling polymorphic
275 // Certain intrinsics have different names than their representative
276 // instructions. This field allows us to handle this correctly when we
277 // are generating tests.
278 string InstName = "";
280 // Certain intrinsics even though they are not a WOpInst or LOpInst,
281 // generate a WOpInst/LOpInst instruction (see below for definition
282 // of a WOpInst/LOpInst). For testing purposes we need to know
283 // this. Ex: vset_lane which outputs vmov instructions.
284 bit isHiddenWInst = 0;
285 bit isHiddenLInst = 0;
288 // The following instruction classes are implemented via builtins.
289 // These declarations are used to generate Builtins.def:
291 // SInst: Instruction with signed/unsigned suffix (e.g., "s8", "u8", "p8")
292 // IInst: Instruction with generic integer suffix (e.g., "i8")
293 // WInst: Instruction with only bit size suffix (e.g., "8")
294 class SInst<string n, string p, string t> : Inst<n, p, t, OP_NONE> {}
295 class IInst<string n, string p, string t> : Inst<n, p, t, OP_NONE> {}
296 class WInst<string n, string p, string t> : Inst<n, p, t, OP_NONE> {}
298 // The following instruction classes are implemented via operators
299 // instead of builtins. As such these declarations are only used for
300 // the purpose of generating tests.
302 // SOpInst: Instruction with signed/unsigned suffix (e.g., "s8",
304 // IOpInst: Instruction with generic integer suffix (e.g., "i8").
305 // WOpInst: Instruction with bit size only suffix (e.g., "8").
306 // LOpInst: Logical instruction with no bit size suffix.
307 // NoTestOpInst: Intrinsic that has no corresponding instruction.
308 class SOpInst<string n, string p, string t, Operation o> : Inst<n, p, t, o> {}
309 class IOpInst<string n, string p, string t, Operation o> : Inst<n, p, t, o> {}
310 class WOpInst<string n, string p, string t, Operation o> : Inst<n, p, t, o> {}
311 class LOpInst<string n, string p, string t, Operation o> : Inst<n, p, t, o> {}
312 class NoTestOpInst<string n, string p, string t, Operation o> : Inst<n, p, t, o> {}
314 //===----------------------------------------------------------------------===//
316 //===----------------------------------------------------------------------===//
318 def OP_ADD : Op<(op "+", $p0, $p1)>;
319 def OP_ADDL : Op<(op "+", (call "vmovl", $p0), (call "vmovl", $p1))>;
320 def OP_ADDLHi : Op<(op "+", (call "vmovl_high", $p0),
321 (call "vmovl_high", $p1))>;
322 def OP_ADDW : Op<(op "+", $p0, (call "vmovl", $p1))>;
323 def OP_ADDWHi : Op<(op "+", $p0, (call "vmovl_high", $p1))>;
324 def OP_SUB : Op<(op "-", $p0, $p1)>;
325 def OP_SUBL : Op<(op "-", (call "vmovl", $p0), (call "vmovl", $p1))>;
326 def OP_SUBLHi : Op<(op "-", (call "vmovl_high", $p0),
327 (call "vmovl_high", $p1))>;
328 def OP_SUBW : Op<(op "-", $p0, (call "vmovl", $p1))>;
329 def OP_SUBWHi : Op<(op "-", $p0, (call "vmovl_high", $p1))>;
330 def OP_MUL : Op<(op "*", $p0, $p1)>;
331 def OP_MLA : Op<(op "+", $p0, (op "*", $p1, $p2))>;
332 def OP_MLAL : Op<(op "+", $p0, (call "vmull", $p1, $p2))>;
333 def OP_MULLHi : Op<(call "vmull", (call "vget_high", $p0),
334 (call "vget_high", $p1))>;
335 def OP_MULLHi_P64 : Op<(call "vmull",
336 (cast "poly64_t", (call "vget_high", $p0)),
337 (cast "poly64_t", (call "vget_high", $p1)))>;
338 def OP_MULLHi_N : Op<(call "vmull_n", (call "vget_high", $p0), $p1)>;
339 def OP_MLALHi : Op<(call "vmlal", $p0, (call "vget_high", $p1),
340 (call "vget_high", $p2))>;
341 def OP_MLALHi_N : Op<(call "vmlal_n", $p0, (call "vget_high", $p1), $p2)>;
342 def OP_MLS : Op<(op "-", $p0, (op "*", $p1, $p2))>;
343 def OP_FMLS : Op<(call "vfma", $p0, (op "-", $p1), $p2)>;
344 def OP_MLSL : Op<(op "-", $p0, (call "vmull", $p1, $p2))>;
345 def OP_MLSLHi : Op<(call "vmlsl", $p0, (call "vget_high", $p1),
346 (call "vget_high", $p2))>;
347 def OP_MLSLHi_N : Op<(call "vmlsl_n", $p0, (call "vget_high", $p1), $p2)>;
348 def OP_MUL_N : Op<(op "*", $p0, (dup $p1))>;
349 def OP_MULX_N : Op<(call "vmulx", $p0, (dup $p1))>;
350 def OP_MLA_N : Op<(op "+", $p0, (op "*", $p1, (dup $p2)))>;
351 def OP_MLS_N : Op<(op "-", $p0, (op "*", $p1, (dup $p2)))>;
352 def OP_FMLA_N : Op<(call "vfma", $p0, $p1, (dup $p2))>;
353 def OP_FMLS_N : Op<(call "vfma", $p0, (op "-", $p1), (dup $p2))>;
354 def OP_MLAL_N : Op<(op "+", $p0, (call "vmull", $p1, (dup $p2)))>;
355 def OP_MLSL_N : Op<(op "-", $p0, (call "vmull", $p1, (dup $p2)))>;
356 def OP_MUL_LN : Op<(op "*", $p0, (splat $p1, $p2))>;
357 def OP_MULX_LN : Op<(call "vmulx", $p0, (splat $p1, $p2))>;
358 def OP_MULL_LN : Op<(call "vmull", $p0, (splat $p1, $p2))>;
359 def OP_MULLHi_LN: Op<(call "vmull", (call "vget_high", $p0), (splat $p1, $p2))>;
360 def OP_MLA_LN : Op<(op "+", $p0, (op "*", $p1, (splat $p2, $p3)))>;
361 def OP_MLS_LN : Op<(op "-", $p0, (op "*", $p1, (splat $p2, $p3)))>;
362 def OP_MLAL_LN : Op<(op "+", $p0, (call "vmull", $p1, (splat $p2, $p3)))>;
363 def OP_MLALHi_LN: Op<(op "+", $p0, (call "vmull", (call "vget_high", $p1),
365 def OP_MLSL_LN : Op<(op "-", $p0, (call "vmull", $p1, (splat $p2, $p3)))>;
366 def OP_MLSLHi_LN : Op<(op "-", $p0, (call "vmull", (call "vget_high", $p1),
368 def OP_QDMULL_LN : Op<(call "vqdmull", $p0, (splat $p1, $p2))>;
369 def OP_QDMULLHi_LN : Op<(call "vqdmull", (call "vget_high", $p0),
371 def OP_QDMLAL_LN : Op<(call "vqdmlal", $p0, $p1, (splat $p2, $p3))>;
372 def OP_QDMLALHi_LN : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
374 def OP_QDMLSL_LN : Op<(call "vqdmlsl", $p0, $p1, (splat $p2, $p3))>;
375 def OP_QDMLSLHi_LN : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
377 def OP_QDMULH_LN : Op<(call "vqdmulh", $p0, (splat $p1, $p2))>;
378 def OP_QRDMULH_LN : Op<(call "vqrdmulh", $p0, (splat $p1, $p2))>;
379 def OP_QRDMLAH : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, $p2))>;
380 def OP_QRDMLSH : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, $p2))>;
381 def OP_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, (splat $p2, $p3)))>;
382 def OP_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (splat $p2, $p3)))>;
383 def OP_FMS_LN : Op<(call "vfma_lane", $p0, (op "-", $p1), $p2, $p3)>;
384 def OP_FMS_LNQ : Op<(call "vfma_laneq", $p0, (op "-", $p1), $p2, $p3)>;
385 def OP_TRN1 : Op<(shuffle $p0, $p1, (interleave (decimate mask0, 2),
386 (decimate mask1, 2)))>;
387 def OP_ZIP1 : Op<(shuffle $p0, $p1, (lowhalf (interleave mask0, mask1)))>;
388 def OP_UZP1 : Op<(shuffle $p0, $p1, (add (decimate mask0, 2),
389 (decimate mask1, 2)))>;
390 def OP_TRN2 : Op<(shuffle $p0, $p1, (interleave
391 (decimate (rotl mask0, 1), 2),
392 (decimate (rotl mask1, 1), 2)))>;
393 def OP_ZIP2 : Op<(shuffle $p0, $p1, (highhalf (interleave mask0, mask1)))>;
394 def OP_UZP2 : Op<(shuffle $p0, $p1, (add (decimate (rotl mask0, 1), 2),
395 (decimate (rotl mask1, 1), 2)))>;
396 def OP_EQ : Op<(cast "R", (op "==", $p0, $p1))>;
397 def OP_GE : Op<(cast "R", (op ">=", $p0, $p1))>;
398 def OP_LE : Op<(cast "R", (op "<=", $p0, $p1))>;
399 def OP_GT : Op<(cast "R", (op ">", $p0, $p1))>;
400 def OP_LT : Op<(cast "R", (op "<", $p0, $p1))>;
401 def OP_NEG : Op<(op "-", $p0)>;
402 def OP_NOT : Op<(op "~", $p0)>;
403 def OP_AND : Op<(op "&", $p0, $p1)>;
404 def OP_OR : Op<(op "|", $p0, $p1)>;
405 def OP_XOR : Op<(op "^", $p0, $p1)>;
406 def OP_ANDN : Op<(op "&", $p0, (op "~", $p1))>;
407 def OP_ORN : Op<(op "|", $p0, (op "~", $p1))>;
408 def OP_CAST : Op<(cast "R", $p0)>;
409 def OP_HI : Op<(shuffle $p0, $p0, (highhalf mask0))>;
410 def OP_LO : Op<(shuffle $p0, $p0, (lowhalf mask0))>;
411 def OP_CONC : Op<(shuffle $p0, $p1, (add mask0, mask1))>;
412 def OP_DUP : Op<(dup $p0)>;
413 def OP_DUP_LN : Op<(splat $p0, $p1)>;
414 def OP_SEL : Op<(cast "R", (op "|",
415 (op "&", $p0, (cast $p0, $p1)),
416 (op "&", (op "~", $p0), (cast $p0, $p2))))>;
417 def OP_REV16 : Op<(shuffle $p0, $p0, (rev 16, mask0))>;
418 def OP_REV32 : Op<(shuffle $p0, $p0, (rev 32, mask0))>;
419 def OP_REV64 : Op<(shuffle $p0, $p0, (rev 64, mask0))>;
420 def OP_XTN : Op<(call "vcombine", $p0, (call "vmovn", $p1))>;
421 def OP_SQXTUN : Op<(call "vcombine", (cast $p0, "U", $p0),
422 (call "vqmovun", $p1))>;
423 def OP_QXTN : Op<(call "vcombine", $p0, (call "vqmovn", $p1))>;
424 def OP_VCVT_NA_HI_F16 : Op<(call "vcombine", $p0, (call "vcvt_f16_f32", $p1))>;
425 def OP_VCVT_NA_HI_F32 : Op<(call "vcombine", $p0, (call "vcvt_f32_f64", $p1))>;
426 def OP_VCVT_EX_HI_F32 : Op<(call "vcvt_f32_f16", (call "vget_high", $p0))>;
427 def OP_VCVT_EX_HI_F64 : Op<(call "vcvt_f64_f32", (call "vget_high", $p0))>;
428 def OP_VCVTX_HI : Op<(call "vcombine", $p0, (call "vcvtx_f32", $p1))>;
429 def OP_REINT : Op<(cast "R", $p0)>;
430 def OP_ADDHNHi : Op<(call "vcombine", $p0, (call "vaddhn", $p1, $p2))>;
431 def OP_RADDHNHi : Op<(call "vcombine", $p0, (call "vraddhn", $p1, $p2))>;
432 def OP_SUBHNHi : Op<(call "vcombine", $p0, (call "vsubhn", $p1, $p2))>;
433 def OP_RSUBHNHi : Op<(call "vcombine", $p0, (call "vrsubhn", $p1, $p2))>;
434 def OP_ABDL : Op<(cast "R", (call "vmovl", (cast $p0, "U",
435 (call "vabd", $p0, $p1))))>;
436 def OP_ABDLHi : Op<(call "vabdl", (call "vget_high", $p0),
437 (call "vget_high", $p1))>;
438 def OP_ABA : Op<(op "+", $p0, (call "vabd", $p1, $p2))>;
439 def OP_ABAL : Op<(op "+", $p0, (call "vabdl", $p1, $p2))>;
440 def OP_ABALHi : Op<(call "vabal", $p0, (call "vget_high", $p1),
441 (call "vget_high", $p2))>;
442 def OP_QDMULLHi : Op<(call "vqdmull", (call "vget_high", $p0),
443 (call "vget_high", $p1))>;
444 def OP_QDMULLHi_N : Op<(call "vqdmull_n", (call "vget_high", $p0), $p1)>;
445 def OP_QDMLALHi : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
446 (call "vget_high", $p2))>;
447 def OP_QDMLALHi_N : Op<(call "vqdmlal_n", $p0, (call "vget_high", $p1), $p2)>;
448 def OP_QDMLSLHi : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
449 (call "vget_high", $p2))>;
450 def OP_QDMLSLHi_N : Op<(call "vqdmlsl_n", $p0, (call "vget_high", $p1), $p2)>;
451 def OP_DIV : Op<(op "/", $p0, $p1)>;
452 def OP_LONG_HI : Op<(cast "R", (call (name_replace "_high_", "_"),
453 (call "vget_high", $p0), $p1))>;
454 def OP_NARROW_HI : Op<(cast "R", (call "vcombine",
455 (cast "R", "H", $p0),
457 (call (name_replace "_high_", "_"),
459 def OP_MOVL_HI : LOp<[(save_temp $a1, (call "vget_high", $p0)),
461 (call "vshll_n", $a1, (literal "int32_t", "0")))]>;
462 def OP_COPY_LN : Op<(call "vset_lane", (call "vget_lane", $p2, $p3), $p0, $p1)>;
463 def OP_SCALAR_MUL_LN : Op<(op "*", $p0, (call "vget_lane", $p1, $p2))>;
464 def OP_SCALAR_MULX_LN : Op<(call "vmulx", $p0, (call "vget_lane", $p1, $p2))>;
465 def OP_SCALAR_VMULX_LN : LOp<[(save_temp $x, (call "vget_lane", $p0,
466 (literal "int32_t", "0"))),
467 (save_temp $y, (call "vget_lane", $p1, $p2)),
468 (save_temp $z, (call "vmulx", $x, $y)),
469 (call "vset_lane", $z, $p0, $p2)]>;
470 def OP_SCALAR_VMULX_LNQ : LOp<[(save_temp $x, (call "vget_lane", $p0,
471 (literal "int32_t", "0"))),
472 (save_temp $y, (call "vget_lane", $p1, $p2)),
473 (save_temp $z, (call "vmulx", $x, $y)),
474 (call "vset_lane", $z, $p0, (literal "int32_t",
476 class ScalarMulOp<string opname> :
477 Op<(call opname, $p0, (call "vget_lane", $p1, $p2))>;
479 def OP_SCALAR_QDMULL_LN : ScalarMulOp<"vqdmull">;
480 def OP_SCALAR_QDMULH_LN : ScalarMulOp<"vqdmulh">;
481 def OP_SCALAR_QRDMULH_LN : ScalarMulOp<"vqrdmulh">;
483 def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1,
484 (call "vget_lane", $p2, $p3)))>;
485 def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1,
486 (call "vget_lane", $p2, $p3)))>;
488 def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t",
490 (bitcast "int16x4_t", $p0), $p1))>;
491 def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t",
493 (bitcast "int16x8_t", $p0), $p1))>;
494 def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t",
496 (bitcast "int16_t", $p0),
497 (bitcast "int16x4_t", $p1), $p2))>;
498 def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t",
500 (bitcast "int16_t", $p0),
501 (bitcast "int16x8_t", $p1), $p2))>;
503 //===----------------------------------------------------------------------===//
505 //===----------------------------------------------------------------------===//
507 ////////////////////////////////////////////////////////////////////////////////
509 def VADD : IOpInst<"vadd", "ddd",
510 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_ADD>;
511 def VADDL : SOpInst<"vaddl", "wdd", "csiUcUsUi", OP_ADDL>;
512 def VADDW : SOpInst<"vaddw", "wwd", "csiUcUsUi", OP_ADDW>;
513 def VHADD : SInst<"vhadd", "ddd", "csiUcUsUiQcQsQiQUcQUsQUi">;
514 def VRHADD : SInst<"vrhadd", "ddd", "csiUcUsUiQcQsQiQUcQUsQUi">;
515 def VQADD : SInst<"vqadd", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
516 def VADDHN : IInst<"vaddhn", "hkk", "silUsUiUl">;
517 def VRADDHN : IInst<"vraddhn", "hkk", "silUsUiUl">;
519 ////////////////////////////////////////////////////////////////////////////////
520 // E.3.2 Multiplication
521 def VMUL : IOpInst<"vmul", "ddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MUL>;
522 def VMULP : SInst<"vmul", "ddd", "PcQPc">;
523 def VMLA : IOpInst<"vmla", "dddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLA>;
524 def VMLAL : SOpInst<"vmlal", "wwdd", "csiUcUsUi", OP_MLAL>;
525 def VMLS : IOpInst<"vmls", "dddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLS>;
526 def VMLSL : SOpInst<"vmlsl", "wwdd", "csiUcUsUi", OP_MLSL>;
527 def VQDMULH : SInst<"vqdmulh", "ddd", "siQsQi">;
528 def VQRDMULH : SInst<"vqrdmulh", "ddd", "siQsQi">;
530 let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in {
531 def VQRDMLAH : SOpInst<"vqrdmlah", "dddd", "siQsQi", OP_QRDMLAH>;
532 def VQRDMLSH : SOpInst<"vqrdmlsh", "dddd", "siQsQi", OP_QRDMLSH>;
535 def VQDMLAL : SInst<"vqdmlal", "wwdd", "si">;
536 def VQDMLSL : SInst<"vqdmlsl", "wwdd", "si">;
537 def VMULL : SInst<"vmull", "wdd", "csiUcUsUiPc">;
538 def VQDMULL : SInst<"vqdmull", "wdd", "si">;
540 ////////////////////////////////////////////////////////////////////////////////
542 def VSUB : IOpInst<"vsub", "ddd",
543 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_SUB>;
544 def VSUBL : SOpInst<"vsubl", "wdd", "csiUcUsUi", OP_SUBL>;
545 def VSUBW : SOpInst<"vsubw", "wwd", "csiUcUsUi", OP_SUBW>;
546 def VQSUB : SInst<"vqsub", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
547 def VHSUB : SInst<"vhsub", "ddd", "csiUcUsUiQcQsQiQUcQUsQUi">;
548 def VSUBHN : IInst<"vsubhn", "hkk", "silUsUiUl">;
549 def VRSUBHN : IInst<"vrsubhn", "hkk", "silUsUiUl">;
551 ////////////////////////////////////////////////////////////////////////////////
553 def VCEQ : IOpInst<"vceq", "udd", "csifUcUsUiPcQcQsQiQfQUcQUsQUiQPc", OP_EQ>;
554 def VCGE : SOpInst<"vcge", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GE>;
555 let InstName = "vcge" in
556 def VCLE : SOpInst<"vcle", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LE>;
557 def VCGT : SOpInst<"vcgt", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GT>;
558 let InstName = "vcgt" in
559 def VCLT : SOpInst<"vclt", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LT>;
560 let InstName = "vacge" in {
561 def VCAGE : IInst<"vcage", "udd", "fQf">;
562 def VCALE : IInst<"vcale", "udd", "fQf">;
564 let InstName = "vacgt" in {
565 def VCAGT : IInst<"vcagt", "udd", "fQf">;
566 def VCALT : IInst<"vcalt", "udd", "fQf">;
568 def VTST : WInst<"vtst", "udd", "csiUcUsUiPcPsQcQsQiQUcQUsQUiQPcQPs">;
570 ////////////////////////////////////////////////////////////////////////////////
571 // E.3.5 Absolute Difference
572 def VABD : SInst<"vabd", "ddd", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
573 def VABDL : SOpInst<"vabdl", "wdd", "csiUcUsUi", OP_ABDL>;
574 def VABA : SOpInst<"vaba", "dddd", "csiUcUsUiQcQsQiQUcQUsQUi", OP_ABA>;
575 def VABAL : SOpInst<"vabal", "wwdd", "csiUcUsUi", OP_ABAL>;
577 ////////////////////////////////////////////////////////////////////////////////
579 def VMAX : SInst<"vmax", "ddd", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
580 def VMIN : SInst<"vmin", "ddd", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
582 ////////////////////////////////////////////////////////////////////////////////
583 // E.3.7 Pairwise Addition
584 def VPADD : IInst<"vpadd", "ddd", "csiUcUsUif">;
585 def VPADDL : SInst<"vpaddl", "nd", "csiUcUsUiQcQsQiQUcQUsQUi">;
586 def VPADAL : SInst<"vpadal", "nnd", "csiUcUsUiQcQsQiQUcQUsQUi">;
588 ////////////////////////////////////////////////////////////////////////////////
589 // E.3.8-9 Folding Max/Min
590 def VPMAX : SInst<"vpmax", "ddd", "csiUcUsUif">;
591 def VPMIN : SInst<"vpmin", "ddd", "csiUcUsUif">;
593 ////////////////////////////////////////////////////////////////////////////////
594 // E.3.10 Reciprocal/Sqrt
595 def VRECPS : IInst<"vrecps", "ddd", "fQf">;
596 def VRSQRTS : IInst<"vrsqrts", "ddd", "fQf">;
598 ////////////////////////////////////////////////////////////////////////////////
599 // E.3.11 Shifts by signed variable
600 def VSHL : SInst<"vshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
601 def VQSHL : SInst<"vqshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
602 def VRSHL : SInst<"vrshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
603 def VQRSHL : SInst<"vqrshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
605 ////////////////////////////////////////////////////////////////////////////////
606 // E.3.12 Shifts by constant
608 def VSHR_N : SInst<"vshr_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
609 def VSHL_N : IInst<"vshl_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
610 def VRSHR_N : SInst<"vrshr_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
611 def VSRA_N : SInst<"vsra_n", "dddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
612 def VRSRA_N : SInst<"vrsra_n", "dddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
613 def VQSHL_N : SInst<"vqshl_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
614 def VQSHLU_N : SInst<"vqshlu_n", "udi", "csilQcQsQiQl">;
615 def VSHRN_N : IInst<"vshrn_n", "hki", "silUsUiUl">;
616 def VQSHRUN_N : SInst<"vqshrun_n", "eki", "sil">;
617 def VQRSHRUN_N : SInst<"vqrshrun_n", "eki", "sil">;
618 def VQSHRN_N : SInst<"vqshrn_n", "hki", "silUsUiUl">;
619 def VRSHRN_N : IInst<"vrshrn_n", "hki", "silUsUiUl">;
620 def VQRSHRN_N : SInst<"vqrshrn_n", "hki", "silUsUiUl">;
621 def VSHLL_N : SInst<"vshll_n", "wdi", "csiUcUsUi">;
623 ////////////////////////////////////////////////////////////////////////////////
624 // E.3.13 Shifts with insert
625 def VSRI_N : WInst<"vsri_n", "dddi",
626 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">;
627 def VSLI_N : WInst<"vsli_n", "dddi",
628 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">;
631 ////////////////////////////////////////////////////////////////////////////////
632 // E.3.14 Loads and stores of a single vector
633 def VLD1 : WInst<"vld1", "dc",
634 "QUcQUsQUiQUlQcQsQiQlQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
635 def VLD1_LANE : WInst<"vld1_lane", "dcdi",
636 "QUcQUsQUiQUlQcQsQiQlQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
637 def VLD1_DUP : WInst<"vld1_dup", "dc",
638 "QUcQUsQUiQUlQcQsQiQlQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
639 def VST1 : WInst<"vst1", "vpd",
640 "QUcQUsQUiQUlQcQsQiQlQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
641 def VST1_LANE : WInst<"vst1_lane", "vpdi",
642 "QUcQUsQUiQUlQcQsQiQlQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
644 ////////////////////////////////////////////////////////////////////////////////
645 // E.3.15 Loads and stores of an N-element structure
646 def VLD2 : WInst<"vld2", "2c", "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
647 def VLD3 : WInst<"vld3", "3c", "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
648 def VLD4 : WInst<"vld4", "4c", "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
649 def VLD2_DUP : WInst<"vld2_dup", "2c", "UcUsUiUlcsilhfPcPs">;
650 def VLD3_DUP : WInst<"vld3_dup", "3c", "UcUsUiUlcsilhfPcPs">;
651 def VLD4_DUP : WInst<"vld4_dup", "4c", "UcUsUiUlcsilhfPcPs">;
652 def VLD2_LANE : WInst<"vld2_lane", "2c2i", "QUsQUiQsQiQhQfQPsUcUsUicsihfPcPs">;
653 def VLD3_LANE : WInst<"vld3_lane", "3c3i", "QUsQUiQsQiQhQfQPsUcUsUicsihfPcPs">;
654 def VLD4_LANE : WInst<"vld4_lane", "4c4i", "QUsQUiQsQiQhQfQPsUcUsUicsihfPcPs">;
655 def VST2 : WInst<"vst2", "vp2", "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
656 def VST3 : WInst<"vst3", "vp3", "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
657 def VST4 : WInst<"vst4", "vp4", "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPs">;
658 def VST2_LANE : WInst<"vst2_lane", "vp2i", "QUsQUiQsQiQhQfQPsUcUsUicsihfPcPs">;
659 def VST3_LANE : WInst<"vst3_lane", "vp3i", "QUsQUiQsQiQhQfQPsUcUsUicsihfPcPs">;
660 def VST4_LANE : WInst<"vst4_lane", "vp4i", "QUsQUiQsQiQhQfQPsUcUsUicsihfPcPs">;
662 ////////////////////////////////////////////////////////////////////////////////
663 // E.3.16 Extract lanes from a vector
664 let InstName = "vmov" in
665 def VGET_LANE : IInst<"vget_lane", "sdi",
666 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">;
668 ////////////////////////////////////////////////////////////////////////////////
669 // E.3.17 Set lanes within a vector
670 let InstName = "vmov" in
671 def VSET_LANE : IInst<"vset_lane", "dsdi",
672 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">;
674 ////////////////////////////////////////////////////////////////////////////////
675 // E.3.18 Initialize a vector from bit pattern
676 def VCREATE : NoTestOpInst<"vcreate", "dl", "csihfUcUsUiUlPcPsl", OP_CAST> {
677 let BigEndianSafe = 1;
680 ////////////////////////////////////////////////////////////////////////////////
681 // E.3.19 Set all lanes to same value
682 let InstName = "vmov" in {
683 def VDUP_N : WOpInst<"vdup_n", "ds",
684 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl",
686 def VMOV_N : WOpInst<"vmov_n", "ds",
687 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl",
691 def VDUP_LANE: WOpInst<"vdup_lane", "dgi",
692 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl",
695 ////////////////////////////////////////////////////////////////////////////////
696 // E.3.20 Combining vectors
697 def VCOMBINE : NoTestOpInst<"vcombine", "kdd", "csilhfUcUsUiUlPcPs", OP_CONC>;
699 ////////////////////////////////////////////////////////////////////////////////
700 // E.3.21 Splitting vectors
701 let InstName = "vmov" in {
702 def VGET_HIGH : NoTestOpInst<"vget_high", "dk", "csilhfUcUsUiUlPcPs", OP_HI>;
703 def VGET_LOW : NoTestOpInst<"vget_low", "dk", "csilhfUcUsUiUlPcPs", OP_LO>;
706 ////////////////////////////////////////////////////////////////////////////////
707 // E.3.22 Converting vectors
709 let ArchGuard = "(__ARM_FP & 2)" in {
710 def VCVT_F16_F32 : SInst<"vcvt_f16_f32", "md", "Hf">;
711 def VCVT_F32_F16 : SInst<"vcvt_f32_f16", "wd", "h">;
714 def VCVT_S32 : SInst<"vcvt_s32", "xd", "fQf">;
715 def VCVT_U32 : SInst<"vcvt_u32", "ud", "fQf">;
716 def VCVT_F32 : SInst<"vcvt_f32", "fd", "iUiQiQUi">;
717 let isVCVT_N = 1 in {
718 def VCVT_N_S32 : SInst<"vcvt_n_s32", "xdi", "fQf">;
719 def VCVT_N_U32 : SInst<"vcvt_n_u32", "udi", "fQf">;
720 def VCVT_N_F32 : SInst<"vcvt_n_f32", "fdi", "iUiQiQUi">;
723 def VMOVN : IInst<"vmovn", "hk", "silUsUiUl">;
724 def VMOVL : SInst<"vmovl", "wd", "csiUcUsUi">;
725 def VQMOVN : SInst<"vqmovn", "hk", "silUsUiUl">;
726 def VQMOVUN : SInst<"vqmovun", "ek", "sil">;
728 ////////////////////////////////////////////////////////////////////////////////
729 // E.3.23-24 Table lookup, Extended table lookup
730 let InstName = "vtbl" in {
731 def VTBL1 : WInst<"vtbl1", "ddt", "UccPc">;
732 def VTBL2 : WInst<"vtbl2", "d2t", "UccPc">;
733 def VTBL3 : WInst<"vtbl3", "d3t", "UccPc">;
734 def VTBL4 : WInst<"vtbl4", "d4t", "UccPc">;
736 let InstName = "vtbx" in {
737 def VTBX1 : WInst<"vtbx1", "dddt", "UccPc">;
738 def VTBX2 : WInst<"vtbx2", "dd2t", "UccPc">;
739 def VTBX3 : WInst<"vtbx3", "dd3t", "UccPc">;
740 def VTBX4 : WInst<"vtbx4", "dd4t", "UccPc">;
743 ////////////////////////////////////////////////////////////////////////////////
744 // E.3.25 Operations with a scalar value
745 def VMLA_LANE : IOpInst<"vmla_lane", "dddgi",
746 "siUsUifQsQiQUsQUiQf", OP_MLA_LN>;
747 def VMLAL_LANE : SOpInst<"vmlal_lane", "wwddi", "siUsUi", OP_MLAL_LN>;
748 def VQDMLAL_LANE : SOpInst<"vqdmlal_lane", "wwddi", "si", OP_QDMLAL_LN>;
749 def VMLS_LANE : IOpInst<"vmls_lane", "dddgi",
750 "siUsUifQsQiQUsQUiQf", OP_MLS_LN>;
751 def VMLSL_LANE : SOpInst<"vmlsl_lane", "wwddi", "siUsUi", OP_MLSL_LN>;
752 def VQDMLSL_LANE : SOpInst<"vqdmlsl_lane", "wwddi", "si", OP_QDMLSL_LN>;
753 def VMUL_N : IOpInst<"vmul_n", "dds", "sifUsUiQsQiQfQUsQUi", OP_MUL_N>;
754 def VMUL_LANE : IOpInst<"vmul_lane", "ddgi",
755 "sifUsUiQsQiQfQUsQUi", OP_MUL_LN>;
756 def VMULL_N : SInst<"vmull_n", "wda", "siUsUi">;
757 def VMULL_LANE : SOpInst<"vmull_lane", "wddi", "siUsUi", OP_MULL_LN>;
758 def VQDMULL_N : SInst<"vqdmull_n", "wda", "si">;
759 def VQDMULL_LANE : SOpInst<"vqdmull_lane", "wddi", "si", OP_QDMULL_LN>;
760 def VQDMULH_N : SInst<"vqdmulh_n", "dda", "siQsQi">;
761 def VQDMULH_LANE : SOpInst<"vqdmulh_lane", "ddgi", "siQsQi", OP_QDMULH_LN>;
762 def VQRDMULH_N : SInst<"vqrdmulh_n", "dda", "siQsQi">;
763 def VQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "ddgi", "siQsQi", OP_QRDMULH_LN>;
765 let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in {
766 def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "dddgi", "siQsQi", OP_QRDMLAH_LN>;
767 def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "dddgi", "siQsQi", OP_QRDMLSH_LN>;
770 def VMLA_N : IOpInst<"vmla_n", "ddda", "siUsUifQsQiQUsQUiQf", OP_MLA_N>;
771 def VMLAL_N : SOpInst<"vmlal_n", "wwda", "siUsUi", OP_MLAL_N>;
772 def VQDMLAL_N : SInst<"vqdmlal_n", "wwda", "si">;
773 def VMLS_N : IOpInst<"vmls_n", "ddds", "siUsUifQsQiQUsQUiQf", OP_MLS_N>;
774 def VMLSL_N : SOpInst<"vmlsl_n", "wwda", "siUsUi", OP_MLSL_N>;
775 def VQDMLSL_N : SInst<"vqdmlsl_n", "wwda", "si">;
777 ////////////////////////////////////////////////////////////////////////////////
778 // E.3.26 Vector Extract
779 def VEXT : WInst<"vext", "dddi",
780 "cUcPcsUsPsiUilUlfQcQUcQPcQsQUsQPsQiQUiQlQUlQf">;
782 ////////////////////////////////////////////////////////////////////////////////
783 // E.3.27 Reverse vector elements
784 def VREV64 : WOpInst<"vrev64", "dd", "csiUcUsUiPcPsfQcQsQiQUcQUsQUiQPcQPsQf",
786 def VREV32 : WOpInst<"vrev32", "dd", "csUcUsPcPsQcQsQUcQUsQPcQPs", OP_REV32>;
787 def VREV16 : WOpInst<"vrev16", "dd", "cUcPcQcQUcQPc", OP_REV16>;
789 ////////////////////////////////////////////////////////////////////////////////
790 // E.3.28 Other single operand arithmetic
791 def VABS : SInst<"vabs", "dd", "csifQcQsQiQf">;
792 def VQABS : SInst<"vqabs", "dd", "csiQcQsQi">;
793 def VNEG : SOpInst<"vneg", "dd", "csifQcQsQiQf", OP_NEG>;
794 def VQNEG : SInst<"vqneg", "dd", "csiQcQsQi">;
795 def VCLS : SInst<"vcls", "dd", "csiQcQsQi">;
796 def VCLZ : IInst<"vclz", "dd", "csiUcUsUiQcQsQiQUcQUsQUi">;
797 def VCNT : WInst<"vcnt", "dd", "UccPcQUcQcQPc">;
798 def VRECPE : SInst<"vrecpe", "dd", "fUiQfQUi">;
799 def VRSQRTE : SInst<"vrsqrte", "dd", "fUiQfQUi">;
801 ////////////////////////////////////////////////////////////////////////////////
802 // E.3.29 Logical operations
803 def VMVN : LOpInst<"vmvn", "dd", "csiUcUsUiPcQcQsQiQUcQUsQUiQPc", OP_NOT>;
804 def VAND : LOpInst<"vand", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_AND>;
805 def VORR : LOpInst<"vorr", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_OR>;
806 def VEOR : LOpInst<"veor", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_XOR>;
807 def VBIC : LOpInst<"vbic", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ANDN>;
808 def VORN : LOpInst<"vorn", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ORN>;
809 let isHiddenLInst = 1 in
810 def VBSL : SInst<"vbsl", "dudd",
811 "csilUcUsUiUlfPcPsQcQsQiQlQUcQUsQUiQUlQfQPcQPs">;
813 ////////////////////////////////////////////////////////////////////////////////
814 // E.3.30 Transposition operations
815 def VTRN : WInst<"vtrn", "2dd", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
816 def VZIP : WInst<"vzip", "2dd", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
817 def VUZP : WInst<"vuzp", "2dd", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
819 ////////////////////////////////////////////////////////////////////////////////
820 // E.3.31 Vector reinterpret cast operations
822 : NoTestOpInst<"vreinterpret", "dd",
823 "csilUcUsUiUlhfPcPsQcQsQiQlQUcQUsQUiQUlQhQfQPcQPs", OP_REINT> {
824 let CartesianProductOfTypes = 1;
825 let ArchGuard = "!defined(__aarch64__)";
826 let BigEndianSafe = 1;
829 ////////////////////////////////////////////////////////////////////////////////
830 // Vector fused multiply-add operations
832 let ArchGuard = "defined(__ARM_FEATURE_FMA)" in {
833 def VFMA : SInst<"vfma", "dddd", "fQf">;
834 def VFMS : SOpInst<"vfms", "dddd", "fQf", OP_FMLS>;
837 ////////////////////////////////////////////////////////////////////////////////
838 // fp16 vector operations
839 def SCALAR_HALF_GET_LANE : IOpInst<"vget_lane", "sdi", "h", OP_SCALAR_HALF_GET_LN>;
840 def SCALAR_HALF_SET_LANE : IOpInst<"vset_lane", "dsdi", "h", OP_SCALAR_HALF_SET_LN>;
841 def SCALAR_HALF_GET_LANEQ : IOpInst<"vget_lane", "sdi", "Qh", OP_SCALAR_HALF_GET_LNQ>;
842 def SCALAR_HALF_SET_LANEQ : IOpInst<"vset_lane", "dsdi", "Qh", OP_SCALAR_HALF_SET_LNQ>;
844 ////////////////////////////////////////////////////////////////////////////////
845 // AArch64 Intrinsics
847 let ArchGuard = "defined(__aarch64__)" in {
849 ////////////////////////////////////////////////////////////////////////////////
851 def LD1 : WInst<"vld1", "dc", "dQdPlQPl">;
852 def LD2 : WInst<"vld2", "2c", "QUlQldQdPlQPl">;
853 def LD3 : WInst<"vld3", "3c", "QUlQldQdPlQPl">;
854 def LD4 : WInst<"vld4", "4c", "QUlQldQdPlQPl">;
855 def ST1 : WInst<"vst1", "vpd", "dQdPlQPl">;
856 def ST2 : WInst<"vst2", "vp2", "QUlQldQdPlQPl">;
857 def ST3 : WInst<"vst3", "vp3", "QUlQldQdPlQPl">;
858 def ST4 : WInst<"vst4", "vp4", "QUlQldQdPlQPl">;
860 def LD1_X2 : WInst<"vld1_x2", "2c",
861 "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPsQUlQldQdPlQPl">;
862 def LD3_x3 : WInst<"vld1_x3", "3c",
863 "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPsQUlQldQdPlQPl">;
864 def LD4_x4 : WInst<"vld1_x4", "4c",
865 "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPsQUlQldQdPlQPl">;
867 def ST1_X2 : WInst<"vst1_x2", "vp2",
868 "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPsQUlQldQdPlQPl">;
869 def ST1_X3 : WInst<"vst1_x3", "vp3",
870 "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPsQUlQldQdPlQPl">;
871 def ST1_X4 : WInst<"vst1_x4", "vp4",
872 "QUcQUsQUiQcQsQiQhQfQPcQPsUcUsUiUlcsilhfPcPsQUlQldQdPlQPl">;
874 def LD1_LANE : WInst<"vld1_lane", "dcdi", "dQdPlQPl">;
875 def LD2_LANE : WInst<"vld2_lane", "2c2i", "lUlQcQUcQPcQlQUldQdPlQPl">;
876 def LD3_LANE : WInst<"vld3_lane", "3c3i", "lUlQcQUcQPcQlQUldQdPlQPl">;
877 def LD4_LANE : WInst<"vld4_lane", "4c4i", "lUlQcQUcQPcQlQUldQdPlQPl">;
878 def ST1_LANE : WInst<"vst1_lane", "vpdi", "dQdPlQPl">;
879 def ST2_LANE : WInst<"vst2_lane", "vp2i", "lUlQcQUcQPcQlQUldQdPlQPl">;
880 def ST3_LANE : WInst<"vst3_lane", "vp3i", "lUlQcQUcQPcQlQUldQdPlQPl">;
881 def ST4_LANE : WInst<"vst4_lane", "vp4i", "lUlQcQUcQPcQlQUldQdPlQPl">;
883 def LD1_DUP : WInst<"vld1_dup", "dc", "dQdPlQPl">;
884 def LD2_DUP : WInst<"vld2_dup", "2c",
885 "QUcQUsQUiQUlQcQsQiQlQhQfQdQPcQPsQPldPl">;
886 def LD3_DUP : WInst<"vld3_dup", "3c",
887 "QUcQUsQUiQUlQcQsQiQlQhQfQdQPcQPsQPldPl">;
888 def LD4_DUP : WInst<"vld4_dup", "4c",
889 "QUcQUsQUiQUlQcQsQiQlQhQfQdQPcQPsQPldPl">;
891 def VLDRQ : WInst<"vldrq", "sc", "Pk">;
892 def VSTRQ : WInst<"vstrq", "vps", "Pk">;
894 ////////////////////////////////////////////////////////////////////////////////
896 def ADD : IOpInst<"vadd", "ddd", "dQd", OP_ADD>;
898 ////////////////////////////////////////////////////////////////////////////////
900 def SUB : IOpInst<"vsub", "ddd", "dQd", OP_SUB>;
902 ////////////////////////////////////////////////////////////////////////////////
904 def MUL : IOpInst<"vmul", "ddd", "dQd", OP_MUL>;
905 def MLA : IOpInst<"vmla", "dddd", "dQd", OP_MLA>;
906 def MLS : IOpInst<"vmls", "dddd", "dQd", OP_MLS>;
908 ////////////////////////////////////////////////////////////////////////////////
909 // Multiplication Extended
910 def MULX : SInst<"vmulx", "ddd", "fdQfQd">;
912 ////////////////////////////////////////////////////////////////////////////////
914 def FDIV : IOpInst<"vdiv", "ddd", "fdQfQd", OP_DIV>;
916 ////////////////////////////////////////////////////////////////////////////////
917 // Vector fused multiply-add operations
918 def FMLA : SInst<"vfma", "dddd", "dQd">;
919 def FMLS : SOpInst<"vfms", "dddd", "dQd", OP_FMLS>;
921 ////////////////////////////////////////////////////////////////////////////////
922 // MUL, MLA, MLS, FMA, FMS definitions with scalar argument
923 def VMUL_N_A64 : IOpInst<"vmul_n", "dds", "Qd", OP_MUL_N>;
925 def FMLA_N : SOpInst<"vfma_n", "ddds", "fQfQd", OP_FMLA_N>;
926 def FMLS_N : SOpInst<"vfms_n", "ddds", "fQfQd", OP_FMLS_N>;
928 def MLA_N : SOpInst<"vmla_n", "ddds", "Qd", OP_MLA_N>;
929 def MLS_N : SOpInst<"vmls_n", "ddds", "Qd", OP_MLS_N>;
931 ////////////////////////////////////////////////////////////////////////////////
932 // Logical operations
933 def BSL : SInst<"vbsl", "dudd", "dPlQdQPl">;
935 ////////////////////////////////////////////////////////////////////////////////
936 // Absolute Difference
937 def ABD : SInst<"vabd", "ddd", "dQd">;
939 ////////////////////////////////////////////////////////////////////////////////
940 // saturating absolute/negate
941 def ABS : SInst<"vabs", "dd", "dQdlQl">;
942 def QABS : SInst<"vqabs", "dd", "lQl">;
943 def NEG : SOpInst<"vneg", "dd", "dlQdQl", OP_NEG>;
944 def QNEG : SInst<"vqneg", "dd", "lQl">;
946 ////////////////////////////////////////////////////////////////////////////////
947 // Signed Saturating Accumulated of Unsigned Value
948 def SUQADD : SInst<"vuqadd", "ddd", "csilQcQsQiQl">;
950 ////////////////////////////////////////////////////////////////////////////////
951 // Unsigned Saturating Accumulated of Signed Value
952 def USQADD : SInst<"vsqadd", "ddd", "UcUsUiUlQUcQUsQUiQUl">;
954 ////////////////////////////////////////////////////////////////////////////////
956 def FRECPS : IInst<"vrecps", "ddd", "dQd">;
957 def FRSQRTS : IInst<"vrsqrts", "ddd", "dQd">;
958 def FRECPE : SInst<"vrecpe", "dd", "dQd">;
959 def FRSQRTE : SInst<"vrsqrte", "dd", "dQd">;
960 def FSQRT : SInst<"vsqrt", "dd", "fdQfQd">;
962 ////////////////////////////////////////////////////////////////////////////////
964 def RBIT : IInst<"vrbit", "dd", "cUcPcQcQUcQPc">;
966 ////////////////////////////////////////////////////////////////////////////////
967 // Integer extract and narrow to high
968 def XTN2 : SOpInst<"vmovn_high", "qhk", "silUsUiUl", OP_XTN>;
970 ////////////////////////////////////////////////////////////////////////////////
971 // Signed integer saturating extract and unsigned narrow to high
972 def SQXTUN2 : SOpInst<"vqmovun_high", "emd", "HsHiHl", OP_SQXTUN>;
974 ////////////////////////////////////////////////////////////////////////////////
975 // Integer saturating extract and narrow to high
976 def QXTN2 : SOpInst<"vqmovn_high", "qhk", "silUsUiUl", OP_QXTN>;
978 ////////////////////////////////////////////////////////////////////////////////
979 // Converting vectors
981 def VCVT_F32_F64 : SInst<"vcvt_f32_f64", "md", "Qd">;
982 def VCVT_F64_F32 : SInst<"vcvt_f64_f32", "wd", "f">;
984 def VCVT_S64 : SInst<"vcvt_s64", "xd", "dQd">;
985 def VCVT_U64 : SInst<"vcvt_u64", "ud", "dQd">;
986 def VCVT_F64 : SInst<"vcvt_f64", "Fd", "lUlQlQUl">;
988 def VCVT_HIGH_F16_F32 : SOpInst<"vcvt_high_f16", "hmj", "Hf", OP_VCVT_NA_HI_F16>;
989 def VCVT_HIGH_F32_F16 : SOpInst<"vcvt_high_f32", "wk", "h", OP_VCVT_EX_HI_F32>;
990 def VCVT_HIGH_F32_F64 : SOpInst<"vcvt_high_f32", "qfj", "d", OP_VCVT_NA_HI_F32>;
991 def VCVT_HIGH_F64_F32 : SOpInst<"vcvt_high_f64", "wj", "f", OP_VCVT_EX_HI_F64>;
993 def VCVTX_F32_F64 : SInst<"vcvtx_f32", "fj", "d">;
994 def VCVTX_HIGH_F32_F64 : SOpInst<"vcvtx_high_f32", "qfj", "d", OP_VCVTX_HI>;
996 ////////////////////////////////////////////////////////////////////////////////
998 def FCAGE : IInst<"vcage", "udd", "dQd">;
999 def FCAGT : IInst<"vcagt", "udd", "dQd">;
1000 def FCALE : IInst<"vcale", "udd", "dQd">;
1001 def FCALT : IInst<"vcalt", "udd", "dQd">;
1002 def CMTST : WInst<"vtst", "udd", "lUlPlQlQUlQPl">;
1003 def CFMEQ : SOpInst<"vceq", "udd", "lUldQdQlQUlPlQPl", OP_EQ>;
1004 def CFMGE : SOpInst<"vcge", "udd", "lUldQdQlQUl", OP_GE>;
1005 def CFMLE : SOpInst<"vcle", "udd", "lUldQdQlQUl", OP_LE>;
1006 def CFMGT : SOpInst<"vcgt", "udd", "lUldQdQlQUl", OP_GT>;
1007 def CFMLT : SOpInst<"vclt", "udd", "lUldQdQlQUl", OP_LT>;
1009 def CMEQ : SInst<"vceqz", "ud",
1010 "csilfUcUsUiUlPcPsPlQcQsQiQlQfQUcQUsQUiQUlQPcQPsdQdQPl">;
1011 def CMGE : SInst<"vcgez", "ud", "csilfdQcQsQiQlQfQd">;
1012 def CMLE : SInst<"vclez", "ud", "csilfdQcQsQiQlQfQd">;
1013 def CMGT : SInst<"vcgtz", "ud", "csilfdQcQsQiQlQfQd">;
1014 def CMLT : SInst<"vcltz", "ud", "csilfdQcQsQiQlQfQd">;
1016 ////////////////////////////////////////////////////////////////////////////////
1018 def MAX : SInst<"vmax", "ddd", "dQd">;
1019 def MIN : SInst<"vmin", "ddd", "dQd">;
1021 ////////////////////////////////////////////////////////////////////////////////
1023 def MAXP : SInst<"vpmax", "ddd", "QcQsQiQUcQUsQUiQfQd">;
1024 def MINP : SInst<"vpmin", "ddd", "QcQsQiQUcQUsQUiQfQd">;
1026 ////////////////////////////////////////////////////////////////////////////////
1027 // Pairwise MaxNum/MinNum Floating Point
1028 def FMAXNMP : SInst<"vpmaxnm", "ddd", "fQfQd">;
1029 def FMINNMP : SInst<"vpminnm", "ddd", "fQfQd">;
1031 ////////////////////////////////////////////////////////////////////////////////
1032 // Pairwise Addition
1033 def ADDP : IInst<"vpadd", "ddd", "QcQsQiQlQUcQUsQUiQUlQfQd">;
1035 ////////////////////////////////////////////////////////////////////////////////
1036 // Shifts by constant
1037 let isShift = 1 in {
1038 // Left shift long high
1039 def SHLL_HIGH_N : SOpInst<"vshll_high_n", "ndi", "HcHsHiHUcHUsHUi",
1042 ////////////////////////////////////////////////////////////////////////////////
1043 def SRI_N : WInst<"vsri_n", "dddi", "PlQPl">;
1044 def SLI_N : WInst<"vsli_n", "dddi", "PlQPl">;
1046 // Right shift narrow high
1047 def SHRN_HIGH_N : IOpInst<"vshrn_high_n", "hmdi",
1048 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
1049 def QSHRUN_HIGH_N : SOpInst<"vqshrun_high_n", "hmdi",
1050 "HsHiHl", OP_NARROW_HI>;
1051 def RSHRN_HIGH_N : IOpInst<"vrshrn_high_n", "hmdi",
1052 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
1053 def QRSHRUN_HIGH_N : SOpInst<"vqrshrun_high_n", "hmdi",
1054 "HsHiHl", OP_NARROW_HI>;
1055 def QSHRN_HIGH_N : SOpInst<"vqshrn_high_n", "hmdi",
1056 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
1057 def QRSHRN_HIGH_N : SOpInst<"vqrshrn_high_n", "hmdi",
1058 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
1061 ////////////////////////////////////////////////////////////////////////////////
1062 // Converting vectors
1063 def VMOVL_HIGH : SOpInst<"vmovl_high", "nd", "HcHsHiHUcHUsHUi", OP_MOVL_HI>;
1065 let isVCVT_N = 1 in {
1066 def CVTF_N_F64 : SInst<"vcvt_n_f64", "Fdi", "lUlQlQUl">;
1067 def FCVTZS_N_S64 : SInst<"vcvt_n_s64", "xdi", "dQd">;
1068 def FCVTZS_N_U64 : SInst<"vcvt_n_u64", "udi", "dQd">;
1071 ////////////////////////////////////////////////////////////////////////////////
1072 // 3VDiff class using high 64-bit in operands
1073 def VADDL_HIGH : SOpInst<"vaddl_high", "wkk", "csiUcUsUi", OP_ADDLHi>;
1074 def VADDW_HIGH : SOpInst<"vaddw_high", "wwk", "csiUcUsUi", OP_ADDWHi>;
1075 def VSUBL_HIGH : SOpInst<"vsubl_high", "wkk", "csiUcUsUi", OP_SUBLHi>;
1076 def VSUBW_HIGH : SOpInst<"vsubw_high", "wwk", "csiUcUsUi", OP_SUBWHi>;
1078 def VABDL_HIGH : SOpInst<"vabdl_high", "wkk", "csiUcUsUi", OP_ABDLHi>;
1079 def VABAL_HIGH : SOpInst<"vabal_high", "wwkk", "csiUcUsUi", OP_ABALHi>;
1081 def VMULL_HIGH : SOpInst<"vmull_high", "wkk", "csiUcUsUiPc", OP_MULLHi>;
1082 def VMULL_HIGH_N : SOpInst<"vmull_high_n", "wks", "siUsUi", OP_MULLHi_N>;
1083 def VMLAL_HIGH : SOpInst<"vmlal_high", "wwkk", "csiUcUsUi", OP_MLALHi>;
1084 def VMLAL_HIGH_N : SOpInst<"vmlal_high_n", "wwks", "siUsUi", OP_MLALHi_N>;
1085 def VMLSL_HIGH : SOpInst<"vmlsl_high", "wwkk", "csiUcUsUi", OP_MLSLHi>;
1086 def VMLSL_HIGH_N : SOpInst<"vmlsl_high_n", "wwks", "siUsUi", OP_MLSLHi_N>;
1088 def VADDHN_HIGH : SOpInst<"vaddhn_high", "qhkk", "silUsUiUl", OP_ADDHNHi>;
1089 def VRADDHN_HIGH : SOpInst<"vraddhn_high", "qhkk", "silUsUiUl", OP_RADDHNHi>;
1090 def VSUBHN_HIGH : SOpInst<"vsubhn_high", "qhkk", "silUsUiUl", OP_SUBHNHi>;
1091 def VRSUBHN_HIGH : SOpInst<"vrsubhn_high", "qhkk", "silUsUiUl", OP_RSUBHNHi>;
1093 def VQDMULL_HIGH : SOpInst<"vqdmull_high", "wkk", "si", OP_QDMULLHi>;
1094 def VQDMULL_HIGH_N : SOpInst<"vqdmull_high_n", "wks", "si", OP_QDMULLHi_N>;
1095 def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "wwkk", "si", OP_QDMLALHi>;
1096 def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "wwks", "si", OP_QDMLALHi_N>;
1097 def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "wwkk", "si", OP_QDMLSLHi>;
1098 def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "wwks", "si", OP_QDMLSLHi_N>;
1099 def VMULL_P64 : SInst<"vmull", "rss", "Pl">;
1100 def VMULL_HIGH_P64 : SOpInst<"vmull_high", "rdd", "HPl", OP_MULLHi_P64>;
1103 ////////////////////////////////////////////////////////////////////////////////
1104 // Extract or insert element from vector
1105 def GET_LANE : IInst<"vget_lane", "sdi", "dQdPlQPl">;
1106 def SET_LANE : IInst<"vset_lane", "dsdi", "dQdPlQPl">;
1107 def COPY_LANE : IOpInst<"vcopy_lane", "ddidi",
1108 "csilUcUsUiUlPcPsPlfd", OP_COPY_LN>;
1109 def COPYQ_LANE : IOpInst<"vcopy_lane", "ddigi",
1110 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN>;
1111 def COPY_LANEQ : IOpInst<"vcopy_laneq", "ddiki",
1112 "csilPcPsPlUcUsUiUlfd", OP_COPY_LN>;
1113 def COPYQ_LANEQ : IOpInst<"vcopy_laneq", "ddidi",
1114 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN>;
1116 ////////////////////////////////////////////////////////////////////////////////
1117 // Set all lanes to same value
1118 def VDUP_LANE1: WOpInst<"vdup_lane", "dgi", "hdQhQdPlQPl", OP_DUP_LN>;
1119 def VDUP_LANE2: WOpInst<"vdup_laneq", "dji",
1120 "csilUcUsUiUlPcPshfdQcQsQiQlQPcQPsQUcQUsQUiQUlQhQfQdPlQPl",
1122 def DUP_N : WOpInst<"vdup_n", "ds", "dQdPlQPl", OP_DUP>;
1123 def MOV_N : WOpInst<"vmov_n", "ds", "dQdPlQPl", OP_DUP>;
1125 ////////////////////////////////////////////////////////////////////////////////
1126 def COMBINE : NoTestOpInst<"vcombine", "kdd", "dPl", OP_CONC>;
1128 ////////////////////////////////////////////////////////////////////////////////
1129 //Initialize a vector from bit pattern
1130 def CREATE : NoTestOpInst<"vcreate", "dl", "dPl", OP_CAST> {
1131 let BigEndianSafe = 1;
1134 ////////////////////////////////////////////////////////////////////////////////
1136 def VMLA_LANEQ : IOpInst<"vmla_laneq", "dddji",
1137 "siUsUifQsQiQUsQUiQf", OP_MLA_LN>;
1138 def VMLS_LANEQ : IOpInst<"vmls_laneq", "dddji",
1139 "siUsUifQsQiQUsQUiQf", OP_MLS_LN>;
1141 def VFMA_LANE : IInst<"vfma_lane", "dddgi", "fdQfQd">;
1142 def VFMA_LANEQ : IInst<"vfma_laneq", "dddji", "fdQfQd"> {
1145 def VFMS_LANE : IOpInst<"vfms_lane", "dddgi", "fdQfQd", OP_FMS_LN>;
1146 def VFMS_LANEQ : IOpInst<"vfms_laneq", "dddji", "fdQfQd", OP_FMS_LNQ>;
1148 def VMLAL_LANEQ : SOpInst<"vmlal_laneq", "wwdki", "siUsUi", OP_MLAL_LN>;
1149 def VMLAL_HIGH_LANE : SOpInst<"vmlal_high_lane", "wwkdi", "siUsUi",
1151 def VMLAL_HIGH_LANEQ : SOpInst<"vmlal_high_laneq", "wwkki", "siUsUi",
1153 def VMLSL_LANEQ : SOpInst<"vmlsl_laneq", "wwdki", "siUsUi", OP_MLSL_LN>;
1154 def VMLSL_HIGH_LANE : SOpInst<"vmlsl_high_lane", "wwkdi", "siUsUi",
1156 def VMLSL_HIGH_LANEQ : SOpInst<"vmlsl_high_laneq", "wwkki", "siUsUi",
1159 def VQDMLAL_LANEQ : SOpInst<"vqdmlal_laneq", "wwdki", "si", OP_QDMLAL_LN>;
1160 def VQDMLAL_HIGH_LANE : SOpInst<"vqdmlal_high_lane", "wwkdi", "si",
1162 def VQDMLAL_HIGH_LANEQ : SOpInst<"vqdmlal_high_laneq", "wwkki", "si",
1164 def VQDMLSL_LANEQ : SOpInst<"vqdmlsl_laneq", "wwdki", "si", OP_QDMLSL_LN>;
1165 def VQDMLSL_HIGH_LANE : SOpInst<"vqdmlsl_high_lane", "wwkdi", "si",
1167 def VQDMLSL_HIGH_LANEQ : SOpInst<"vqdmlsl_high_laneq", "wwkki", "si",
1170 // Newly add double parameter for vmul_lane in aarch64
1171 // Note: d type is handled by SCALAR_VMUL_LANE
1172 def VMUL_LANE_A64 : IOpInst<"vmul_lane", "ddgi", "Qd", OP_MUL_LN>;
1174 // Note: d type is handled by SCALAR_VMUL_LANEQ
1175 def VMUL_LANEQ : IOpInst<"vmul_laneq", "ddji",
1176 "sifUsUiQsQiQUsQUiQfQd", OP_MUL_LN>;
1177 def VMULL_LANEQ : SOpInst<"vmull_laneq", "wdki", "siUsUi", OP_MULL_LN>;
1178 def VMULL_HIGH_LANE : SOpInst<"vmull_high_lane", "wkdi", "siUsUi",
1180 def VMULL_HIGH_LANEQ : SOpInst<"vmull_high_laneq", "wkki", "siUsUi",
1183 def VQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "wdki", "si", OP_QDMULL_LN>;
1184 def VQDMULL_HIGH_LANE : SOpInst<"vqdmull_high_lane", "wkdi", "si",
1186 def VQDMULL_HIGH_LANEQ : SOpInst<"vqdmull_high_laneq", "wkki", "si",
1189 def VQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "ddji", "siQsQi", OP_QDMULH_LN>;
1190 def VQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "ddji", "siQsQi", OP_QRDMULH_LN>;
1192 let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1193 def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "dddji", "siQsQi", OP_QRDMLAH_LN>;
1194 def VQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "dddji", "siQsQi", OP_QRDMLSH_LN>;
1197 // Note: d type implemented by SCALAR_VMULX_LANE
1198 def VMULX_LANE : IOpInst<"vmulx_lane", "ddgi", "fQfQd", OP_MULX_LN>;
1199 // Note: d type is implemented by SCALAR_VMULX_LANEQ
1200 def VMULX_LANEQ : IOpInst<"vmulx_laneq", "ddji", "fQfQd", OP_MULX_LN>;
1202 ////////////////////////////////////////////////////////////////////////////////
1203 // Across vectors class
1204 def VADDLV : SInst<"vaddlv", "rd", "csiUcUsUiQcQsQiQUcQUsQUi">;
1205 def VMAXV : SInst<"vmaxv", "sd", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">;
1206 def VMINV : SInst<"vminv", "sd", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">;
1207 def VADDV : SInst<"vaddv", "sd", "csifUcUsUiQcQsQiQUcQUsQUiQfQdQlQUl">;
1208 def FMAXNMV : SInst<"vmaxnmv", "sd", "fQfQd">;
1209 def FMINNMV : SInst<"vminnmv", "sd", "fQfQd">;
1211 ////////////////////////////////////////////////////////////////////////////////
1212 // Newly added Vector Extract for f64
1213 def VEXT_A64 : WInst<"vext", "dddi", "dQdPlQPl">;
1215 ////////////////////////////////////////////////////////////////////////////////
1217 let ArchGuard = "__ARM_FEATURE_CRYPTO" in {
1218 def AESE : SInst<"vaese", "ddd", "QUc">;
1219 def AESD : SInst<"vaesd", "ddd", "QUc">;
1220 def AESMC : SInst<"vaesmc", "dd", "QUc">;
1221 def AESIMC : SInst<"vaesimc", "dd", "QUc">;
1223 def SHA1H : SInst<"vsha1h", "ss", "Ui">;
1224 def SHA1SU1 : SInst<"vsha1su1", "ddd", "QUi">;
1225 def SHA256SU0 : SInst<"vsha256su0", "ddd", "QUi">;
1227 def SHA1C : SInst<"vsha1c", "ddsd", "QUi">;
1228 def SHA1P : SInst<"vsha1p", "ddsd", "QUi">;
1229 def SHA1M : SInst<"vsha1m", "ddsd", "QUi">;
1230 def SHA1SU0 : SInst<"vsha1su0", "dddd", "QUi">;
1231 def SHA256H : SInst<"vsha256h", "dddd", "QUi">;
1232 def SHA256H2 : SInst<"vsha256h2", "dddd", "QUi">;
1233 def SHA256SU1 : SInst<"vsha256su1", "dddd", "QUi">;
1236 ////////////////////////////////////////////////////////////////////////////////
1237 // Float -> Int conversions with explicit rounding mode
1239 let ArchGuard = "__ARM_ARCH >= 8" in {
1240 def FCVTNS_S32 : SInst<"vcvtn_s32", "xd", "fQf">;
1241 def FCVTNU_S32 : SInst<"vcvtn_u32", "ud", "fQf">;
1242 def FCVTPS_S32 : SInst<"vcvtp_s32", "xd", "fQf">;
1243 def FCVTPU_S32 : SInst<"vcvtp_u32", "ud", "fQf">;
1244 def FCVTMS_S32 : SInst<"vcvtm_s32", "xd", "fQf">;
1245 def FCVTMU_S32 : SInst<"vcvtm_u32", "ud", "fQf">;
1246 def FCVTAS_S32 : SInst<"vcvta_s32", "xd", "fQf">;
1247 def FCVTAU_S32 : SInst<"vcvta_u32", "ud", "fQf">;
1250 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)" in {
1251 def FCVTNS_S64 : SInst<"vcvtn_s64", "xd", "dQd">;
1252 def FCVTNU_S64 : SInst<"vcvtn_u64", "ud", "dQd">;
1253 def FCVTPS_S64 : SInst<"vcvtp_s64", "xd", "dQd">;
1254 def FCVTPU_S64 : SInst<"vcvtp_u64", "ud", "dQd">;
1255 def FCVTMS_S64 : SInst<"vcvtm_s64", "xd", "dQd">;
1256 def FCVTMU_S64 : SInst<"vcvtm_u64", "ud", "dQd">;
1257 def FCVTAS_S64 : SInst<"vcvta_s64", "xd", "dQd">;
1258 def FCVTAU_S64 : SInst<"vcvta_u64", "ud", "dQd">;
1261 ////////////////////////////////////////////////////////////////////////////////
1262 // Round to Integral
1264 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1265 def FRINTN_S32 : SInst<"vrndn", "dd", "fQf">;
1266 def FRINTA_S32 : SInst<"vrnda", "dd", "fQf">;
1267 def FRINTP_S32 : SInst<"vrndp", "dd", "fQf">;
1268 def FRINTM_S32 : SInst<"vrndm", "dd", "fQf">;
1269 def FRINTX_S32 : SInst<"vrndx", "dd", "fQf">;
1270 def FRINTZ_S32 : SInst<"vrnd", "dd", "fQf">;
1273 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1274 def FRINTN_S64 : SInst<"vrndn", "dd", "dQd">;
1275 def FRINTA_S64 : SInst<"vrnda", "dd", "dQd">;
1276 def FRINTP_S64 : SInst<"vrndp", "dd", "dQd">;
1277 def FRINTM_S64 : SInst<"vrndm", "dd", "dQd">;
1278 def FRINTX_S64 : SInst<"vrndx", "dd", "dQd">;
1279 def FRINTZ_S64 : SInst<"vrnd", "dd", "dQd">;
1280 def FRINTI_S64 : SInst<"vrndi", "dd", "fdQfQd">;
1283 ////////////////////////////////////////////////////////////////////////////////
1284 // MaxNum/MinNum Floating Point
1286 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in {
1287 def FMAXNM_S32 : SInst<"vmaxnm", "ddd", "fQf">;
1288 def FMINNM_S32 : SInst<"vminnm", "ddd", "fQf">;
1291 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in {
1292 def FMAXNM_S64 : SInst<"vmaxnm", "ddd", "dQd">;
1293 def FMINNM_S64 : SInst<"vminnm", "ddd", "dQd">;
1296 ////////////////////////////////////////////////////////////////////////////////
1298 def VTRN1 : SOpInst<"vtrn1", "ddd",
1299 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN1>;
1300 def VZIP1 : SOpInst<"vzip1", "ddd",
1301 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP1>;
1302 def VUZP1 : SOpInst<"vuzp1", "ddd",
1303 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP1>;
1304 def VTRN2 : SOpInst<"vtrn2", "ddd",
1305 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN2>;
1306 def VZIP2 : SOpInst<"vzip2", "ddd",
1307 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP2>;
1308 def VUZP2 : SOpInst<"vuzp2", "ddd",
1309 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP2>;
1311 ////////////////////////////////////////////////////////////////////////////////
1313 let InstName = "vtbl" in {
1314 def VQTBL1_A64 : WInst<"vqtbl1", "djt", "UccPcQUcQcQPc">;
1315 def VQTBL2_A64 : WInst<"vqtbl2", "dBt", "UccPcQUcQcQPc">;
1316 def VQTBL3_A64 : WInst<"vqtbl3", "dCt", "UccPcQUcQcQPc">;
1317 def VQTBL4_A64 : WInst<"vqtbl4", "dDt", "UccPcQUcQcQPc">;
1319 let InstName = "vtbx" in {
1320 def VQTBX1_A64 : WInst<"vqtbx1", "ddjt", "UccPcQUcQcQPc">;
1321 def VQTBX2_A64 : WInst<"vqtbx2", "ddBt", "UccPcQUcQcQPc">;
1322 def VQTBX3_A64 : WInst<"vqtbx3", "ddCt", "UccPcQUcQcQPc">;
1323 def VQTBX4_A64 : WInst<"vqtbx4", "ddDt", "UccPcQUcQcQPc">;
1326 ////////////////////////////////////////////////////////////////////////////////
1327 // Vector reinterpret cast operations
1329 // NeonEmitter implicitly takes the cartesian product of the type string with
1330 // itself during generation so, unlike all other intrinsics, this one should
1331 // include *all* types, not just additional ones.
1333 : NoTestOpInst<"vreinterpret", "dd",
1334 "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", OP_REINT> {
1335 let CartesianProductOfTypes = 1;
1336 let BigEndianSafe = 1;
1337 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)";
1340 ////////////////////////////////////////////////////////////////////////////////
1341 // Scalar Intrinsics
1342 // Scalar Arithmetic
1345 def SCALAR_ADD : SInst<"vadd", "sss", "SlSUl">;
1346 // Scalar Saturating Add
1347 def SCALAR_QADD : SInst<"vqadd", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1349 // Scalar Subtraction
1350 def SCALAR_SUB : SInst<"vsub", "sss", "SlSUl">;
1351 // Scalar Saturating Sub
1352 def SCALAR_QSUB : SInst<"vqsub", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1354 let InstName = "vmov" in {
1355 def VGET_HIGH_A64 : NoTestOpInst<"vget_high", "dk", "dPl", OP_HI>;
1356 def VGET_LOW_A64 : NoTestOpInst<"vget_low", "dk", "dPl", OP_LO>;
1359 ////////////////////////////////////////////////////////////////////////////////
1361 // Scalar Shift Left
1362 def SCALAR_SHL: SInst<"vshl", "sss", "SlSUl">;
1363 // Scalar Saturating Shift Left
1364 def SCALAR_QSHL: SInst<"vqshl", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1365 // Scalar Saturating Rounding Shift Left
1366 def SCALAR_QRSHL: SInst<"vqrshl", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1367 // Scalar Shift Rounding Left
1368 def SCALAR_RSHL: SInst<"vrshl", "sss", "SlSUl">;
1370 ////////////////////////////////////////////////////////////////////////////////
1371 // Scalar Shift (Immediate)
1372 let isScalarShift = 1 in {
1373 // Signed/Unsigned Shift Right (Immediate)
1374 def SCALAR_SSHR_N: SInst<"vshr_n", "ssi", "SlSUl">;
1375 // Signed/Unsigned Rounding Shift Right (Immediate)
1376 def SCALAR_SRSHR_N: SInst<"vrshr_n", "ssi", "SlSUl">;
1378 // Signed/Unsigned Shift Right and Accumulate (Immediate)
1379 def SCALAR_SSRA_N: SInst<"vsra_n", "sssi", "SlSUl">;
1380 // Signed/Unsigned Rounding Shift Right and Accumulate (Immediate)
1381 def SCALAR_SRSRA_N: SInst<"vrsra_n", "sssi", "SlSUl">;
1383 // Shift Left (Immediate)
1384 def SCALAR_SHL_N: SInst<"vshl_n", "ssi", "SlSUl">;
1385 // Signed/Unsigned Saturating Shift Left (Immediate)
1386 def SCALAR_SQSHL_N: SInst<"vqshl_n", "ssi", "ScSsSiSlSUcSUsSUiSUl">;
1387 // Signed Saturating Shift Left Unsigned (Immediate)
1388 def SCALAR_SQSHLU_N: SInst<"vqshlu_n", "ssi", "ScSsSiSl">;
1390 // Shift Right And Insert (Immediate)
1391 def SCALAR_SRI_N: SInst<"vsri_n", "sssi", "SlSUl">;
1392 // Shift Left And Insert (Immediate)
1393 def SCALAR_SLI_N: SInst<"vsli_n", "sssi", "SlSUl">;
1395 let isScalarNarrowShift = 1 in {
1396 // Signed/Unsigned Saturating Shift Right Narrow (Immediate)
1397 def SCALAR_SQSHRN_N: SInst<"vqshrn_n", "zsi", "SsSiSlSUsSUiSUl">;
1398 // Signed/Unsigned Saturating Rounded Shift Right Narrow (Immediate)
1399 def SCALAR_SQRSHRN_N: SInst<"vqrshrn_n", "zsi", "SsSiSlSUsSUiSUl">;
1400 // Signed Saturating Shift Right Unsigned Narrow (Immediate)
1401 def SCALAR_SQSHRUN_N: SInst<"vqshrun_n", "zsi", "SsSiSl">;
1402 // Signed Saturating Rounded Shift Right Unsigned Narrow (Immediate)
1403 def SCALAR_SQRSHRUN_N: SInst<"vqrshrun_n", "zsi", "SsSiSl">;
1406 ////////////////////////////////////////////////////////////////////////////////
1407 // Scalar Signed/Unsigned Fixed-point Convert To Floating-Point (Immediate)
1408 def SCALAR_SCVTF_N_F32: SInst<"vcvt_n_f32", "ysi", "SiSUi">;
1409 def SCALAR_SCVTF_N_F64: SInst<"vcvt_n_f64", "osi", "SlSUl">;
1411 ////////////////////////////////////////////////////////////////////////////////
1412 // Scalar Floating-point Convert To Signed/Unsigned Fixed-point (Immediate)
1413 def SCALAR_FCVTZS_N_S32 : SInst<"vcvt_n_s32", "$si", "Sf">;
1414 def SCALAR_FCVTZU_N_U32 : SInst<"vcvt_n_u32", "bsi", "Sf">;
1415 def SCALAR_FCVTZS_N_S64 : SInst<"vcvt_n_s64", "$si", "Sd">;
1416 def SCALAR_FCVTZU_N_U64 : SInst<"vcvt_n_u64", "bsi", "Sd">;
1419 ////////////////////////////////////////////////////////////////////////////////
1420 // Scalar Reduce Pairwise Addition (Scalar and Floating Point)
1421 def SCALAR_ADDP : SInst<"vpadd", "sd", "SfSHlSHdSHUl">;
1423 ////////////////////////////////////////////////////////////////////////////////
1424 // Scalar Reduce Floating Point Pairwise Max/Min
1425 def SCALAR_FMAXP : SInst<"vpmax", "sd", "SfSQd">;
1427 def SCALAR_FMINP : SInst<"vpmin", "sd", "SfSQd">;
1429 ////////////////////////////////////////////////////////////////////////////////
1430 // Scalar Reduce Floating Point Pairwise maxNum/minNum
1431 def SCALAR_FMAXNMP : SInst<"vpmaxnm", "sd", "SfSQd">;
1432 def SCALAR_FMINNMP : SInst<"vpminnm", "sd", "SfSQd">;
1434 ////////////////////////////////////////////////////////////////////////////////
1435 // Scalar Integer Saturating Doubling Multiply Half High
1436 def SCALAR_SQDMULH : SInst<"vqdmulh", "sss", "SsSi">;
1438 ////////////////////////////////////////////////////////////////////////////////
1439 // Scalar Integer Saturating Rounding Doubling Multiply Half High
1440 def SCALAR_SQRDMULH : SInst<"vqrdmulh", "sss", "SsSi">;
1442 let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1443 ////////////////////////////////////////////////////////////////////////////////
1444 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
1445 def SCALAR_SQRDMLAH : SOpInst<"vqrdmlah", "ssss", "SsSi", OP_QRDMLAH>;
1447 ////////////////////////////////////////////////////////////////////////////////
1448 // Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
1449 def SCALAR_SQRDMLSH : SOpInst<"vqrdmlsh", "ssss", "SsSi", OP_QRDMLSH>;
1452 ////////////////////////////////////////////////////////////////////////////////
1453 // Scalar Floating-point Multiply Extended
1454 def SCALAR_FMULX : IInst<"vmulx", "sss", "SfSd">;
1456 ////////////////////////////////////////////////////////////////////////////////
1457 // Scalar Floating-point Reciprocal Step
1458 def SCALAR_FRECPS : IInst<"vrecps", "sss", "SfSd">;
1460 ////////////////////////////////////////////////////////////////////////////////
1461 // Scalar Floating-point Reciprocal Square Root Step
1462 def SCALAR_FRSQRTS : IInst<"vrsqrts", "sss", "SfSd">;
1464 ////////////////////////////////////////////////////////////////////////////////
1465 // Scalar Signed Integer Convert To Floating-point
1466 def SCALAR_SCVTFS : SInst<"vcvt_f32", "ys", "Si">;
1467 def SCALAR_SCVTFD : SInst<"vcvt_f64", "os", "Sl">;
1469 ////////////////////////////////////////////////////////////////////////////////
1470 // Scalar Unsigned Integer Convert To Floating-point
1471 def SCALAR_UCVTFS : SInst<"vcvt_f32", "ys", "SUi">;
1472 def SCALAR_UCVTFD : SInst<"vcvt_f64", "os", "SUl">;
1474 ////////////////////////////////////////////////////////////////////////////////
1475 // Scalar Floating-point Converts
1476 def SCALAR_FCVTXN : IInst<"vcvtx_f32", "ys", "Sd">;
1477 def SCALAR_FCVTNSS : SInst<"vcvtn_s32", "$s", "Sf">;
1478 def SCALAR_FCVTNUS : SInst<"vcvtn_u32", "bs", "Sf">;
1479 def SCALAR_FCVTNSD : SInst<"vcvtn_s64", "$s", "Sd">;
1480 def SCALAR_FCVTNUD : SInst<"vcvtn_u64", "bs", "Sd">;
1481 def SCALAR_FCVTMSS : SInst<"vcvtm_s32", "$s", "Sf">;
1482 def SCALAR_FCVTMUS : SInst<"vcvtm_u32", "bs", "Sf">;
1483 def SCALAR_FCVTMSD : SInst<"vcvtm_s64", "$s", "Sd">;
1484 def SCALAR_FCVTMUD : SInst<"vcvtm_u64", "bs", "Sd">;
1485 def SCALAR_FCVTASS : SInst<"vcvta_s32", "$s", "Sf">;
1486 def SCALAR_FCVTAUS : SInst<"vcvta_u32", "bs", "Sf">;
1487 def SCALAR_FCVTASD : SInst<"vcvta_s64", "$s", "Sd">;
1488 def SCALAR_FCVTAUD : SInst<"vcvta_u64", "bs", "Sd">;
1489 def SCALAR_FCVTPSS : SInst<"vcvtp_s32", "$s", "Sf">;
1490 def SCALAR_FCVTPUS : SInst<"vcvtp_u32", "bs", "Sf">;
1491 def SCALAR_FCVTPSD : SInst<"vcvtp_s64", "$s", "Sd">;
1492 def SCALAR_FCVTPUD : SInst<"vcvtp_u64", "bs", "Sd">;
1493 def SCALAR_FCVTZSS : SInst<"vcvt_s32", "$s", "Sf">;
1494 def SCALAR_FCVTZUS : SInst<"vcvt_u32", "bs", "Sf">;
1495 def SCALAR_FCVTZSD : SInst<"vcvt_s64", "$s", "Sd">;
1496 def SCALAR_FCVTZUD : SInst<"vcvt_u64", "bs", "Sd">;
1498 ////////////////////////////////////////////////////////////////////////////////
1499 // Scalar Floating-point Reciprocal Estimate
1500 def SCALAR_FRECPE : IInst<"vrecpe", "ss", "SfSd">;
1502 ////////////////////////////////////////////////////////////////////////////////
1503 // Scalar Floating-point Reciprocal Exponent
1504 def SCALAR_FRECPX : IInst<"vrecpx", "ss", "SfSd">;
1506 ////////////////////////////////////////////////////////////////////////////////
1507 // Scalar Floating-point Reciprocal Square Root Estimate
1508 def SCALAR_FRSQRTE : IInst<"vrsqrte", "ss", "SfSd">;
1510 ////////////////////////////////////////////////////////////////////////////////
1511 // Scalar Integer Comparison
1512 def SCALAR_CMEQ : SInst<"vceq", "sss", "SlSUl">;
1513 def SCALAR_CMEQZ : SInst<"vceqz", "ss", "SlSUl">;
1514 def SCALAR_CMGE : SInst<"vcge", "sss", "Sl">;
1515 def SCALAR_CMGEZ : SInst<"vcgez", "ss", "Sl">;
1516 def SCALAR_CMHS : SInst<"vcge", "sss", "SUl">;
1517 def SCALAR_CMLE : SInst<"vcle", "sss", "SlSUl">;
1518 def SCALAR_CMLEZ : SInst<"vclez", "ss", "Sl">;
1519 def SCALAR_CMLT : SInst<"vclt", "sss", "SlSUl">;
1520 def SCALAR_CMLTZ : SInst<"vcltz", "ss", "Sl">;
1521 def SCALAR_CMGT : SInst<"vcgt", "sss", "Sl">;
1522 def SCALAR_CMGTZ : SInst<"vcgtz", "ss", "Sl">;
1523 def SCALAR_CMHI : SInst<"vcgt", "sss", "SUl">;
1524 def SCALAR_CMTST : SInst<"vtst", "sss", "SlSUl">;
1526 ////////////////////////////////////////////////////////////////////////////////
1527 // Scalar Floating-point Comparison
1528 def SCALAR_FCMEQ : IInst<"vceq", "bss", "SfSd">;
1529 def SCALAR_FCMEQZ : IInst<"vceqz", "bs", "SfSd">;
1530 def SCALAR_FCMGE : IInst<"vcge", "bss", "SfSd">;
1531 def SCALAR_FCMGEZ : IInst<"vcgez", "bs", "SfSd">;
1532 def SCALAR_FCMGT : IInst<"vcgt", "bss", "SfSd">;
1533 def SCALAR_FCMGTZ : IInst<"vcgtz", "bs", "SfSd">;
1534 def SCALAR_FCMLE : IInst<"vcle", "bss", "SfSd">;
1535 def SCALAR_FCMLEZ : IInst<"vclez", "bs", "SfSd">;
1536 def SCALAR_FCMLT : IInst<"vclt", "bss", "SfSd">;
1537 def SCALAR_FCMLTZ : IInst<"vcltz", "bs", "SfSd">;
1539 ////////////////////////////////////////////////////////////////////////////////
1540 // Scalar Floating-point Absolute Compare Mask Greater Than Or Equal
1541 def SCALAR_FACGE : IInst<"vcage", "bss", "SfSd">;
1542 def SCALAR_FACLE : IInst<"vcale", "bss", "SfSd">;
1544 ////////////////////////////////////////////////////////////////////////////////
1545 // Scalar Floating-point Absolute Compare Mask Greater Than
1546 def SCALAR_FACGT : IInst<"vcagt", "bss", "SfSd">;
1547 def SCALAR_FACLT : IInst<"vcalt", "bss", "SfSd">;
1549 ////////////////////////////////////////////////////////////////////////////////
1550 // Scalar Absolute Value
1551 def SCALAR_ABS : SInst<"vabs", "ss", "Sl">;
1553 ////////////////////////////////////////////////////////////////////////////////
1554 // Scalar Absolute Difference
1555 def SCALAR_ABD : IInst<"vabd", "sss", "SfSd">;
1557 ////////////////////////////////////////////////////////////////////////////////
1558 // Scalar Signed Saturating Absolute Value
1559 def SCALAR_SQABS : SInst<"vqabs", "ss", "ScSsSiSl">;
1561 ////////////////////////////////////////////////////////////////////////////////
1563 def SCALAR_NEG : SInst<"vneg", "ss", "Sl">;
1565 ////////////////////////////////////////////////////////////////////////////////
1566 // Scalar Signed Saturating Negate
1567 def SCALAR_SQNEG : SInst<"vqneg", "ss", "ScSsSiSl">;
1569 ////////////////////////////////////////////////////////////////////////////////
1570 // Scalar Signed Saturating Accumulated of Unsigned Value
1571 def SCALAR_SUQADD : SInst<"vuqadd", "sss", "ScSsSiSl">;
1573 ////////////////////////////////////////////////////////////////////////////////
1574 // Scalar Unsigned Saturating Accumulated of Signed Value
1575 def SCALAR_USQADD : SInst<"vsqadd", "sss", "SUcSUsSUiSUl">;
1577 ////////////////////////////////////////////////////////////////////////////////
1578 // Signed Saturating Doubling Multiply-Add Long
1579 def SCALAR_SQDMLAL : SInst<"vqdmlal", "rrss", "SsSi">;
1581 ////////////////////////////////////////////////////////////////////////////////
1582 // Signed Saturating Doubling Multiply-Subtract Long
1583 def SCALAR_SQDMLSL : SInst<"vqdmlsl", "rrss", "SsSi">;
1585 ////////////////////////////////////////////////////////////////////////////////
1586 // Signed Saturating Doubling Multiply Long
1587 def SCALAR_SQDMULL : SInst<"vqdmull", "rss", "SsSi">;
1589 ////////////////////////////////////////////////////////////////////////////////
1590 // Scalar Signed Saturating Extract Unsigned Narrow
1591 def SCALAR_SQXTUN : SInst<"vqmovun", "zs", "SsSiSl">;
1593 ////////////////////////////////////////////////////////////////////////////////
1594 // Scalar Signed Saturating Extract Narrow
1595 def SCALAR_SQXTN : SInst<"vqmovn", "zs", "SsSiSl">;
1597 ////////////////////////////////////////////////////////////////////////////////
1598 // Scalar Unsigned Saturating Extract Narrow
1599 def SCALAR_UQXTN : SInst<"vqmovn", "zs", "SUsSUiSUl">;
1601 // Scalar Floating Point multiply (scalar, by element)
1602 def SCALAR_FMUL_LANE : IOpInst<"vmul_lane", "ssdi", "SfSd", OP_SCALAR_MUL_LN>;
1603 def SCALAR_FMUL_LANEQ : IOpInst<"vmul_laneq", "ssji", "SfSd", OP_SCALAR_MUL_LN>;
1605 // Scalar Floating Point multiply extended (scalar, by element)
1606 def SCALAR_FMULX_LANE : IOpInst<"vmulx_lane", "ssdi", "SfSd", OP_SCALAR_MULX_LN>;
1607 def SCALAR_FMULX_LANEQ : IOpInst<"vmulx_laneq", "ssji", "SfSd", OP_SCALAR_MULX_LN>;
1609 def SCALAR_VMUL_N : IInst<"vmul_n", "dds", "d">;
1611 // VMUL_LANE_A64 d type implemented using scalar mul lane
1612 def SCALAR_VMUL_LANE : IInst<"vmul_lane", "ddgi", "d">;
1614 // VMUL_LANEQ d type implemented using scalar mul lane
1615 def SCALAR_VMUL_LANEQ : IInst<"vmul_laneq", "ddji", "d"> {
1619 // VMULX_LANE d type implemented using scalar vmulx_lane
1620 def SCALAR_VMULX_LANE : IOpInst<"vmulx_lane", "ddgi", "d", OP_SCALAR_VMULX_LN>;
1622 // VMULX_LANEQ d type implemented using scalar vmulx_laneq
1623 def SCALAR_VMULX_LANEQ : IOpInst<"vmulx_laneq", "ddji", "d", OP_SCALAR_VMULX_LNQ>;
1625 // Scalar Floating Point fused multiply-add (scalar, by element)
1626 def SCALAR_FMLA_LANE : IInst<"vfma_lane", "sssdi", "SfSd">;
1627 def SCALAR_FMLA_LANEQ : IInst<"vfma_laneq", "sssji", "SfSd">;
1629 // Scalar Floating Point fused multiply-subtract (scalar, by element)
1630 def SCALAR_FMLS_LANE : IOpInst<"vfms_lane", "sssdi", "SfSd", OP_FMS_LN>;
1631 def SCALAR_FMLS_LANEQ : IOpInst<"vfms_laneq", "sssji", "SfSd", OP_FMS_LNQ>;
1633 // Signed Saturating Doubling Multiply Long (scalar by element)
1634 def SCALAR_SQDMULL_LANE : SOpInst<"vqdmull_lane", "rsdi", "SsSi", OP_SCALAR_QDMULL_LN>;
1635 def SCALAR_SQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "rsji", "SsSi", OP_SCALAR_QDMULL_LN>;
1637 // Signed Saturating Doubling Multiply-Add Long (scalar by element)
1638 def SCALAR_SQDMLAL_LANE : SInst<"vqdmlal_lane", "rrsdi", "SsSi">;
1639 def SCALAR_SQDMLAL_LANEQ : SInst<"vqdmlal_laneq", "rrsji", "SsSi">;
1641 // Signed Saturating Doubling Multiply-Subtract Long (scalar by element)
1642 def SCALAR_SQDMLS_LANE : SInst<"vqdmlsl_lane", "rrsdi", "SsSi">;
1643 def SCALAR_SQDMLS_LANEQ : SInst<"vqdmlsl_laneq", "rrsji", "SsSi">;
1645 // Scalar Integer Saturating Doubling Multiply Half High (scalar by element)
1646 def SCALAR_SQDMULH_LANE : SOpInst<"vqdmulh_lane", "ssdi", "SsSi", OP_SCALAR_QDMULH_LN>;
1647 def SCALAR_SQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "ssji", "SsSi", OP_SCALAR_QDMULH_LN>;
1649 // Scalar Integer Saturating Rounding Doubling Multiply Half High
1650 def SCALAR_SQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "ssdi", "SsSi", OP_SCALAR_QRDMULH_LN>;
1651 def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "ssji", "SsSi", OP_SCALAR_QRDMULH_LN>;
1653 let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1654 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
1655 def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "sssdi", "SsSi", OP_SCALAR_QRDMLAH_LN>;
1656 def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "sssji", "SsSi", OP_SCALAR_QRDMLAH_LN>;
1658 // Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
1659 def SCALAR_SQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "sssdi", "SsSi", OP_SCALAR_QRDMLSH_LN>;
1660 def SCALAR_SQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "sssji", "SsSi", OP_SCALAR_QRDMLSH_LN>;
1663 def SCALAR_VDUP_LANE : IInst<"vdup_lane", "sdi", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
1664 def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "sji", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
1667 // ARMv8.2-A FP16 intrinsics.
1668 let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in {
1670 // ARMv8.2-A FP16 one-operand vector intrinsics.
1673 def CMEQH : SInst<"vceqz", "ud", "hQh">;
1674 def CMGEH : SInst<"vcgez", "ud", "hQh">;
1675 def CMGTH : SInst<"vcgtz", "ud", "hQh">;
1676 def CMLEH : SInst<"vclez", "ud", "hQh">;
1677 def CMLTH : SInst<"vcltz", "ud", "hQh">;
1679 // Vector conversion
1680 def VCVT_F16 : SInst<"vcvt_f16", "Hd", "sUsQsQUs">;
1681 def VCVT_S16 : SInst<"vcvt_s16", "xd", "hQh">;
1682 def VCVT_U16 : SInst<"vcvt_u16", "ud", "hQh">;
1683 def VCVTA_S16 : SInst<"vcvta_s16", "xd", "hQh">;
1684 def VCVTA_U16 : SInst<"vcvta_u16", "ud", "hQh">;
1685 def VCVTM_S16 : SInst<"vcvtm_s16", "xd", "hQh">;
1686 def VCVTM_U16 : SInst<"vcvtm_u16", "ud", "hQh">;
1687 def VCVTN_S16 : SInst<"vcvtn_s16", "xd", "hQh">;
1688 def VCVTN_U16 : SInst<"vcvtn_u16", "ud", "hQh">;
1689 def VCVTP_S16 : SInst<"vcvtp_s16", "xd", "hQh">;
1690 def VCVTP_U16 : SInst<"vcvtp_u16", "ud", "hQh">;
1693 def FRINTZH : SInst<"vrnd", "dd", "hQh">;
1694 def FRINTNH : SInst<"vrndn", "dd", "hQh">;
1695 def FRINTAH : SInst<"vrnda", "dd", "hQh">;
1696 def FRINTPH : SInst<"vrndp", "dd", "hQh">;
1697 def FRINTMH : SInst<"vrndm", "dd", "hQh">;
1698 def FRINTXH : SInst<"vrndx", "dd", "hQh">;
1699 def FRINTIH : SInst<"vrndi", "dd", "hQh">;
1702 def VABSH : SInst<"vabs", "dd", "hQh">;
1703 def VNEGH : SOpInst<"vneg", "dd", "hQh", OP_NEG>;
1704 def VRECPEH : SInst<"vrecpe", "dd", "hQh">;
1705 def FRSQRTEH : SInst<"vrsqrte", "dd", "hQh">;
1706 def FSQRTH : SInst<"vsqrt", "dd", "hQh">;
1708 // ARMv8.2-A FP16 two-operands vector intrinsics.
1711 def VADDH : SOpInst<"vadd", "ddd", "hQh", OP_ADD>;
1712 def VABDH : SInst<"vabd", "ddd", "hQh">;
1713 def VSUBH : SOpInst<"vsub", "ddd", "hQh", OP_SUB>;
1716 let InstName = "vacge" in {
1717 def VCAGEH : SInst<"vcage", "udd", "hQh">;
1718 def VCALEH : SInst<"vcale", "udd", "hQh">;
1720 let InstName = "vacgt" in {
1721 def VCAGTH : SInst<"vcagt", "udd", "hQh">;
1722 def VCALTH : SInst<"vcalt", "udd", "hQh">;
1724 def VCEQH : SOpInst<"vceq", "udd", "hQh", OP_EQ>;
1725 def VCGEH : SOpInst<"vcge", "udd", "hQh", OP_GE>;
1726 def VCGTH : SOpInst<"vcgt", "udd", "hQh", OP_GT>;
1727 let InstName = "vcge" in
1728 def VCLEH : SOpInst<"vcle", "udd", "hQh", OP_LE>;
1729 let InstName = "vcgt" in
1730 def VCLTH : SOpInst<"vclt", "udd", "hQh", OP_LT>;
1732 // Vector conversion
1733 let isVCVT_N = 1 in {
1734 def VCVT_N_F16 : SInst<"vcvt_n_f16", "Hdi", "sUsQsQUs">;
1735 def VCVT_N_S16 : SInst<"vcvt_n_s16", "xdi", "hQh">;
1736 def VCVT_N_U16 : SInst<"vcvt_n_u16", "udi", "hQh">;
1740 def VMAXH : SInst<"vmax", "ddd", "hQh">;
1741 def VMINH : SInst<"vmin", "ddd", "hQh">;
1742 def FMAXNMH : SInst<"vmaxnm", "ddd", "hQh">;
1743 def FMINNMH : SInst<"vminnm", "ddd", "hQh">;
1745 // Multiplication/Division
1746 def VMULH : SOpInst<"vmul", "ddd", "hQh", OP_MUL>;
1747 def MULXH : SInst<"vmulx", "ddd", "hQh">;
1748 def FDIVH : IOpInst<"vdiv", "ddd", "hQh", OP_DIV>;
1750 // Pairwise addition
1751 def VPADDH : SInst<"vpadd", "ddd", "hQh">;
1754 def VPMAXH : SInst<"vpmax", "ddd", "hQh">;
1755 def VPMINH : SInst<"vpmin", "ddd", "hQh">;
1756 // Pairwise MaxNum/MinNum
1757 def FMAXNMPH : SInst<"vpmaxnm", "ddd", "hQh">;
1758 def FMINNMPH : SInst<"vpminnm", "ddd", "hQh">;
1761 def VRECPSH : SInst<"vrecps", "ddd", "hQh">;
1762 def VRSQRTSH : SInst<"vrsqrts", "ddd", "hQh">;
1764 // ARMv8.2-A FP16 three-operands vector intrinsics.
1766 // Vector fused multiply-add operations
1767 def VFMAH : SInst<"vfma", "dddd", "hQh">;
1768 def VFMSH : SOpInst<"vfms", "dddd", "hQh", OP_FMLS>;
1770 // ARMv8.2-A FP16 lane vector intrinsics.
1773 def VFMA_LANEH : IInst<"vfma_lane", "dddgi", "hQh">;
1774 def VFMA_LANEQH : IInst<"vfma_laneq", "dddji", "hQh">;
1776 // FMA lane with scalar argument
1777 def FMLA_NH : SOpInst<"vfma_n", "ddds", "hQh", OP_FMLA_N>;
1778 // Scalar floating point fused multiply-add (scalar, by element)
1779 def SCALAR_FMLA_LANEH : IInst<"vfma_lane", "sssdi", "Sh">;
1780 def SCALAR_FMLA_LANEQH : IInst<"vfma_laneq", "sssji", "Sh">;
1783 def VFMS_LANEH : IOpInst<"vfms_lane", "dddgi", "hQh", OP_FMS_LN>;
1784 def VFMS_LANEQH : IOpInst<"vfms_laneq", "dddji", "hQh", OP_FMS_LNQ>;
1785 // FMS lane with scalar argument
1786 def FMLS_NH : SOpInst<"vfms_n", "ddds", "hQh", OP_FMLS_N>;
1787 // Scalar floating foint fused multiply-subtract (scalar, by element)
1788 def SCALAR_FMLS_LANEH : IOpInst<"vfms_lane", "sssdi", "Sh", OP_FMS_LN>;
1789 def SCALAR_FMLS_LANEQH : IOpInst<"vfms_laneq", "sssji", "Sh", OP_FMS_LNQ>;
1792 def VMUL_LANEH : IOpInst<"vmul_lane", "ddgi", "hQh", OP_MUL_LN>;
1793 def VMUL_LANEQH : IOpInst<"vmul_laneq", "ddji", "hQh", OP_MUL_LN>;
1794 def VMUL_NH : IOpInst<"vmul_n", "dds", "hQh", OP_MUL_N>;
1795 // Scalar floating point multiply (scalar, by element)
1796 def SCALAR_FMUL_LANEH : IOpInst<"vmul_lane", "ssdi", "Sh", OP_SCALAR_MUL_LN>;
1797 def SCALAR_FMUL_LANEQH : IOpInst<"vmul_laneq", "ssji", "Sh", OP_SCALAR_MUL_LN>;
1800 def VMULX_LANEH : IOpInst<"vmulx_lane", "ddgi", "hQh", OP_MULX_LN>;
1801 def VMULX_LANEQH : IOpInst<"vmulx_laneq", "ddji", "hQh", OP_MULX_LN>;
1802 def VMULX_NH : IOpInst<"vmulx_n", "dds", "hQh", OP_MULX_N>;
1803 // TODO: Scalar floating point multiply extended (scalar, by element)
1804 // Below ones are commented out because they need vmulx_f16(float16_t, float16_t)
1805 // which will be implemented later with fp16 scalar intrinsic (arm_fp16.h)
1806 //def SCALAR_FMULX_LANEH : IOpInst<"vmulx_lane", "ssdi", "Sh", OP_SCALAR_MUL_LN>;
1807 //def SCALAR_FMULX_LANEQH : IOpInst<"vmulx_laneq", "ssji", "Sh", OP_SCALAR_MUL_LN>;
1809 // ARMv8.2-A FP16 reduction vector intrinsics.
1810 def VMAXVH : SInst<"vmaxv", "sd", "hQh">;
1811 def VMINVH : SInst<"vminv", "sd", "hQh">;
1812 def FMAXNMVH : SInst<"vmaxnmv", "sd", "hQh">;
1813 def FMINNMVH : SInst<"vminnmv", "sd", "hQh">;
1815 // Data processing intrinsics - section 5
1817 // Logical operations
1818 let isHiddenLInst = 1 in
1819 def VBSLH : SInst<"vbsl", "dudd", "hQh">;
1821 // Transposition operations
1822 def VZIPH : WInst<"vzip", "2dd", "hQh">;
1823 def VUZPH : WInst<"vuzp", "2dd", "hQh">;
1824 def VTRNH : WInst<"vtrn", "2dd", "hQh">;
1826 // Set all lanes to same value.
1827 /* Already implemented prior to ARMv8.2-A.
1828 def VMOV_NH : WOpInst<"vmov_n", "ds", "hQh", OP_DUP>;
1829 def VDUP_NH : WOpInst<"vdup_n", "ds", "hQh", OP_DUP>;
1830 def VDUP_LANE1H : WOpInst<"vdup_lane", "dgi", "hQh", OP_DUP_LN>;*/
1833 def VEXTH : WInst<"vext", "dddi", "hQh">;
1835 // Reverse vector elements
1836 def VREV64H : WOpInst<"vrev64", "dd", "hQh", OP_REV64>;
1839 def VTRN1H : SOpInst<"vtrn1", "ddd", "hQh", OP_TRN1>;
1840 def VZIP1H : SOpInst<"vzip1", "ddd", "hQh", OP_ZIP1>;
1841 def VUZP1H : SOpInst<"vuzp1", "ddd", "hQh", OP_UZP1>;
1842 def VTRN2H : SOpInst<"vtrn2", "ddd", "hQh", OP_TRN2>;
1843 def VZIP2H : SOpInst<"vzip2", "ddd", "hQh", OP_ZIP2>;
1844 def VUZP2H : SOpInst<"vuzp2", "ddd", "hQh", OP_UZP2>;
1846 def SCALAR_VDUP_LANEH : IInst<"vdup_lane", "sdi", "Sh">;
1847 def SCALAR_VDUP_LANEQH : IInst<"vdup_laneq", "sji", "Sh">;