1 //===--- arm_neon.td - ARM NEON compiler interface ------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the TableGen definitions from which the ARM NEON header
11 // file will be generated. See ARM document DUI0348B.
13 //===----------------------------------------------------------------------===//
15 include "arm_neon_incl.td"
17 def OP_ADD : Op<(op "+", $p0, $p1)>;
18 def OP_ADDL : Op<(op "+", (call "vmovl", $p0), (call "vmovl", $p1))>;
19 def OP_ADDLHi : Op<(op "+", (call "vmovl_high", $p0),
20 (call "vmovl_high", $p1))>;
21 def OP_ADDW : Op<(op "+", $p0, (call "vmovl", $p1))>;
22 def OP_ADDWHi : Op<(op "+", $p0, (call "vmovl_high", $p1))>;
23 def OP_SUB : Op<(op "-", $p0, $p1)>;
24 def OP_SUBL : Op<(op "-", (call "vmovl", $p0), (call "vmovl", $p1))>;
25 def OP_SUBLHi : Op<(op "-", (call "vmovl_high", $p0),
26 (call "vmovl_high", $p1))>;
27 def OP_SUBW : Op<(op "-", $p0, (call "vmovl", $p1))>;
28 def OP_SUBWHi : Op<(op "-", $p0, (call "vmovl_high", $p1))>;
29 def OP_MUL : Op<(op "*", $p0, $p1)>;
30 def OP_MLA : Op<(op "+", $p0, (op "*", $p1, $p2))>;
31 def OP_MLAL : Op<(op "+", $p0, (call "vmull", $p1, $p2))>;
32 def OP_MULLHi : Op<(call "vmull", (call "vget_high", $p0),
33 (call "vget_high", $p1))>;
34 def OP_MULLHi_P64 : Op<(call "vmull",
35 (cast "poly64_t", (call "vget_high", $p0)),
36 (cast "poly64_t", (call "vget_high", $p1)))>;
37 def OP_MULLHi_N : Op<(call "vmull_n", (call "vget_high", $p0), $p1)>;
38 def OP_MLALHi : Op<(call "vmlal", $p0, (call "vget_high", $p1),
39 (call "vget_high", $p2))>;
40 def OP_MLALHi_N : Op<(call "vmlal_n", $p0, (call "vget_high", $p1), $p2)>;
41 def OP_MLS : Op<(op "-", $p0, (op "*", $p1, $p2))>;
42 def OP_FMLS : Op<(call "vfma", $p0, (op "-", $p1), $p2)>;
43 def OP_MLSL : Op<(op "-", $p0, (call "vmull", $p1, $p2))>;
44 def OP_MLSLHi : Op<(call "vmlsl", $p0, (call "vget_high", $p1),
45 (call "vget_high", $p2))>;
46 def OP_MLSLHi_N : Op<(call "vmlsl_n", $p0, (call "vget_high", $p1), $p2)>;
47 def OP_MUL_N : Op<(op "*", $p0, (dup $p1))>;
48 def OP_MULX_N : Op<(call "vmulx", $p0, (dup $p1))>;
49 def OP_MLA_N : Op<(op "+", $p0, (op "*", $p1, (dup $p2)))>;
50 def OP_MLS_N : Op<(op "-", $p0, (op "*", $p1, (dup $p2)))>;
51 def OP_FMLA_N : Op<(call "vfma", $p0, $p1, (dup $p2))>;
52 def OP_FMLS_N : Op<(call "vfma", $p0, (op "-", $p1), (dup $p2))>;
53 def OP_MLAL_N : Op<(op "+", $p0, (call "vmull", $p1, (dup $p2)))>;
54 def OP_MLSL_N : Op<(op "-", $p0, (call "vmull", $p1, (dup $p2)))>;
55 def OP_MUL_LN : Op<(op "*", $p0, (splat $p1, $p2))>;
56 def OP_MULX_LN : Op<(call "vmulx", $p0, (splat $p1, $p2))>;
57 def OP_MULL_LN : Op<(call "vmull", $p0, (splat $p1, $p2))>;
58 def OP_MULLHi_LN: Op<(call "vmull", (call "vget_high", $p0), (splat $p1, $p2))>;
59 def OP_MLA_LN : Op<(op "+", $p0, (op "*", $p1, (splat $p2, $p3)))>;
60 def OP_MLS_LN : Op<(op "-", $p0, (op "*", $p1, (splat $p2, $p3)))>;
61 def OP_MLAL_LN : Op<(op "+", $p0, (call "vmull", $p1, (splat $p2, $p3)))>;
62 def OP_MLALHi_LN: Op<(op "+", $p0, (call "vmull", (call "vget_high", $p1),
64 def OP_MLSL_LN : Op<(op "-", $p0, (call "vmull", $p1, (splat $p2, $p3)))>;
65 def OP_MLSLHi_LN : Op<(op "-", $p0, (call "vmull", (call "vget_high", $p1),
67 def OP_QDMULL_LN : Op<(call "vqdmull", $p0, (splat $p1, $p2))>;
68 def OP_QDMULLHi_LN : Op<(call "vqdmull", (call "vget_high", $p0),
70 def OP_QDMLAL_LN : Op<(call "vqdmlal", $p0, $p1, (splat $p2, $p3))>;
71 def OP_QDMLALHi_LN : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
73 def OP_QDMLSL_LN : Op<(call "vqdmlsl", $p0, $p1, (splat $p2, $p3))>;
74 def OP_QDMLSLHi_LN : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
76 def OP_QDMULH_LN : Op<(call "vqdmulh", $p0, (splat $p1, $p2))>;
77 def OP_QRDMULH_LN : Op<(call "vqrdmulh", $p0, (splat $p1, $p2))>;
78 def OP_QRDMLAH : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, $p2))>;
79 def OP_QRDMLSH : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, $p2))>;
80 def OP_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, (splat $p2, $p3)))>;
81 def OP_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (splat $p2, $p3)))>;
82 def OP_FMS_LN : Op<(call "vfma_lane", $p0, (op "-", $p1), $p2, $p3)>;
83 def OP_FMS_LNQ : Op<(call "vfma_laneq", $p0, (op "-", $p1), $p2, $p3)>;
84 def OP_TRN1 : Op<(shuffle $p0, $p1, (interleave (decimate mask0, 2),
85 (decimate mask1, 2)))>;
86 def OP_ZIP1 : Op<(shuffle $p0, $p1, (lowhalf (interleave mask0, mask1)))>;
87 def OP_UZP1 : Op<(shuffle $p0, $p1, (add (decimate mask0, 2),
88 (decimate mask1, 2)))>;
89 def OP_TRN2 : Op<(shuffle $p0, $p1, (interleave
90 (decimate (rotl mask0, 1), 2),
91 (decimate (rotl mask1, 1), 2)))>;
92 def OP_ZIP2 : Op<(shuffle $p0, $p1, (highhalf (interleave mask0, mask1)))>;
93 def OP_UZP2 : Op<(shuffle $p0, $p1, (add (decimate (rotl mask0, 1), 2),
94 (decimate (rotl mask1, 1), 2)))>;
95 def OP_EQ : Op<(cast "R", (op "==", $p0, $p1))>;
96 def OP_GE : Op<(cast "R", (op ">=", $p0, $p1))>;
97 def OP_LE : Op<(cast "R", (op "<=", $p0, $p1))>;
98 def OP_GT : Op<(cast "R", (op ">", $p0, $p1))>;
99 def OP_LT : Op<(cast "R", (op "<", $p0, $p1))>;
100 def OP_NEG : Op<(op "-", $p0)>;
101 def OP_NOT : Op<(op "~", $p0)>;
102 def OP_AND : Op<(op "&", $p0, $p1)>;
103 def OP_OR : Op<(op "|", $p0, $p1)>;
104 def OP_XOR : Op<(op "^", $p0, $p1)>;
105 def OP_ANDN : Op<(op "&", $p0, (op "~", $p1))>;
106 def OP_ORN : Op<(op "|", $p0, (op "~", $p1))>;
107 def OP_CAST : Op<(cast "R", $p0)>;
108 def OP_HI : Op<(shuffle $p0, $p0, (highhalf mask0))>;
109 def OP_LO : Op<(shuffle $p0, $p0, (lowhalf mask0))>;
110 def OP_CONC : Op<(shuffle $p0, $p1, (add mask0, mask1))>;
111 def OP_DUP : Op<(dup $p0)>;
112 def OP_DUP_LN : Op<(splat $p0, $p1)>;
113 def OP_SEL : Op<(cast "R", (op "|",
114 (op "&", $p0, (cast $p0, $p1)),
115 (op "&", (op "~", $p0), (cast $p0, $p2))))>;
116 def OP_REV16 : Op<(shuffle $p0, $p0, (rev 16, mask0))>;
117 def OP_REV32 : Op<(shuffle $p0, $p0, (rev 32, mask0))>;
118 def OP_REV64 : Op<(shuffle $p0, $p0, (rev 64, mask0))>;
119 def OP_XTN : Op<(call "vcombine", $p0, (call "vmovn", $p1))>;
120 def OP_SQXTUN : Op<(call "vcombine", (cast $p0, "U", $p0),
121 (call "vqmovun", $p1))>;
122 def OP_QXTN : Op<(call "vcombine", $p0, (call "vqmovn", $p1))>;
123 def OP_VCVT_NA_HI_F16 : Op<(call "vcombine", $p0, (call "vcvt_f16_f32", $p1))>;
124 def OP_VCVT_NA_HI_F32 : Op<(call "vcombine", $p0, (call "vcvt_f32_f64", $p1))>;
125 def OP_VCVT_EX_HI_F32 : Op<(call "vcvt_f32_f16", (call "vget_high", $p0))>;
126 def OP_VCVT_EX_HI_F64 : Op<(call "vcvt_f64_f32", (call "vget_high", $p0))>;
127 def OP_VCVTX_HI : Op<(call "vcombine", $p0, (call "vcvtx_f32", $p1))>;
128 def OP_REINT : Op<(cast "R", $p0)>;
129 def OP_ADDHNHi : Op<(call "vcombine", $p0, (call "vaddhn", $p1, $p2))>;
130 def OP_RADDHNHi : Op<(call "vcombine", $p0, (call "vraddhn", $p1, $p2))>;
131 def OP_SUBHNHi : Op<(call "vcombine", $p0, (call "vsubhn", $p1, $p2))>;
132 def OP_RSUBHNHi : Op<(call "vcombine", $p0, (call "vrsubhn", $p1, $p2))>;
133 def OP_ABDL : Op<(cast "R", (call "vmovl", (cast $p0, "U",
134 (call "vabd", $p0, $p1))))>;
135 def OP_ABDLHi : Op<(call "vabdl", (call "vget_high", $p0),
136 (call "vget_high", $p1))>;
137 def OP_ABA : Op<(op "+", $p0, (call "vabd", $p1, $p2))>;
138 def OP_ABAL : Op<(op "+", $p0, (call "vabdl", $p1, $p2))>;
139 def OP_ABALHi : Op<(call "vabal", $p0, (call "vget_high", $p1),
140 (call "vget_high", $p2))>;
141 def OP_QDMULLHi : Op<(call "vqdmull", (call "vget_high", $p0),
142 (call "vget_high", $p1))>;
143 def OP_QDMULLHi_N : Op<(call "vqdmull_n", (call "vget_high", $p0), $p1)>;
144 def OP_QDMLALHi : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
145 (call "vget_high", $p2))>;
146 def OP_QDMLALHi_N : Op<(call "vqdmlal_n", $p0, (call "vget_high", $p1), $p2)>;
147 def OP_QDMLSLHi : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
148 (call "vget_high", $p2))>;
149 def OP_QDMLSLHi_N : Op<(call "vqdmlsl_n", $p0, (call "vget_high", $p1), $p2)>;
150 def OP_DIV : Op<(op "/", $p0, $p1)>;
151 def OP_LONG_HI : Op<(cast "R", (call (name_replace "_high_", "_"),
152 (call "vget_high", $p0), $p1))>;
153 def OP_NARROW_HI : Op<(cast "R", (call "vcombine",
154 (cast "R", "H", $p0),
156 (call (name_replace "_high_", "_"),
158 def OP_MOVL_HI : LOp<[(save_temp $a1, (call "vget_high", $p0)),
160 (call "vshll_n", $a1, (literal "int32_t", "0")))]>;
161 def OP_COPY_LN : Op<(call "vset_lane", (call "vget_lane", $p2, $p3), $p0, $p1)>;
162 def OP_SCALAR_MUL_LN : Op<(op "*", $p0, (call "vget_lane", $p1, $p2))>;
163 def OP_SCALAR_MULX_LN : Op<(call "vmulx", $p0, (call "vget_lane", $p1, $p2))>;
164 def OP_SCALAR_VMULX_LN : LOp<[(save_temp $x, (call "vget_lane", $p0,
165 (literal "int32_t", "0"))),
166 (save_temp $y, (call "vget_lane", $p1, $p2)),
167 (save_temp $z, (call "vmulx", $x, $y)),
168 (call "vset_lane", $z, $p0, $p2)]>;
169 def OP_SCALAR_VMULX_LNQ : LOp<[(save_temp $x, (call "vget_lane", $p0,
170 (literal "int32_t", "0"))),
171 (save_temp $y, (call "vget_lane", $p1, $p2)),
172 (save_temp $z, (call "vmulx", $x, $y)),
173 (call "vset_lane", $z, $p0, (literal "int32_t",
175 class ScalarMulOp<string opname> :
176 Op<(call opname, $p0, (call "vget_lane", $p1, $p2))>;
178 def OP_SCALAR_QDMULL_LN : ScalarMulOp<"vqdmull">;
179 def OP_SCALAR_QDMULH_LN : ScalarMulOp<"vqdmulh">;
180 def OP_SCALAR_QRDMULH_LN : ScalarMulOp<"vqrdmulh">;
182 def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1,
183 (call "vget_lane", $p2, $p3)))>;
184 def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1,
185 (call "vget_lane", $p2, $p3)))>;
187 def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t",
189 (bitcast "int16x4_t", $p0), $p1))>;
190 def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t",
192 (bitcast "int16x8_t", $p0), $p1))>;
193 def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t",
195 (bitcast "int16_t", $p0),
196 (bitcast "int16x4_t", $p1), $p2))>;
197 def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t",
199 (bitcast "int16_t", $p0),
200 (bitcast "int16x8_t", $p1), $p2))>;
203 : Op<(call "vdot", $p0, $p1,
204 (bitcast $p1, (splat(bitcast "uint32x2_t", $p2), $p3)))>;
206 : Op<(call "vdot", $p0, $p1,
207 (bitcast $p1, (splat(bitcast "uint32x4_t", $p2), $p3)))>;
209 def OP_FMLAL_LN : Op<(call "vfmlal_low", $p0, $p1,
210 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
211 def OP_FMLSL_LN : Op<(call "vfmlsl_low", $p0, $p1,
212 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
213 def OP_FMLAL_LN_Hi : Op<(call "vfmlal_high", $p0, $p1,
214 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
215 def OP_FMLSL_LN_Hi : Op<(call "vfmlsl_high", $p0, $p1,
216 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
218 //===----------------------------------------------------------------------===//
220 //===----------------------------------------------------------------------===//
222 ////////////////////////////////////////////////////////////////////////////////
224 def VADD : IOpInst<"vadd", "ddd",
225 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_ADD>;
226 def VADDL : SOpInst<"vaddl", "wdd", "csiUcUsUi", OP_ADDL>;
227 def VADDW : SOpInst<"vaddw", "wwd", "csiUcUsUi", OP_ADDW>;
228 def VHADD : SInst<"vhadd", "ddd", "csiUcUsUiQcQsQiQUcQUsQUi">;
229 def VRHADD : SInst<"vrhadd", "ddd", "csiUcUsUiQcQsQiQUcQUsQUi">;
230 def VQADD : SInst<"vqadd", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
231 def VADDHN : IInst<"vaddhn", "hkk", "silUsUiUl">;
232 def VRADDHN : IInst<"vraddhn", "hkk", "silUsUiUl">;
234 ////////////////////////////////////////////////////////////////////////////////
235 // E.3.2 Multiplication
236 def VMUL : IOpInst<"vmul", "ddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MUL>;
237 def VMULP : SInst<"vmul", "ddd", "PcQPc">;
238 def VMLA : IOpInst<"vmla", "dddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLA>;
239 def VMLAL : SOpInst<"vmlal", "wwdd", "csiUcUsUi", OP_MLAL>;
240 def VMLS : IOpInst<"vmls", "dddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLS>;
241 def VMLSL : SOpInst<"vmlsl", "wwdd", "csiUcUsUi", OP_MLSL>;
242 def VQDMULH : SInst<"vqdmulh", "ddd", "siQsQi">;
243 def VQRDMULH : SInst<"vqrdmulh", "ddd", "siQsQi">;
245 let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in {
246 def VQRDMLAH : SOpInst<"vqrdmlah", "dddd", "siQsQi", OP_QRDMLAH>;
247 def VQRDMLSH : SOpInst<"vqrdmlsh", "dddd", "siQsQi", OP_QRDMLSH>;
250 def VQDMLAL : SInst<"vqdmlal", "wwdd", "si">;
251 def VQDMLSL : SInst<"vqdmlsl", "wwdd", "si">;
252 def VMULL : SInst<"vmull", "wdd", "csiUcUsUiPc">;
253 def VQDMULL : SInst<"vqdmull", "wdd", "si">;
255 ////////////////////////////////////////////////////////////////////////////////
257 def VSUB : IOpInst<"vsub", "ddd",
258 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_SUB>;
259 def VSUBL : SOpInst<"vsubl", "wdd", "csiUcUsUi", OP_SUBL>;
260 def VSUBW : SOpInst<"vsubw", "wwd", "csiUcUsUi", OP_SUBW>;
261 def VQSUB : SInst<"vqsub", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
262 def VHSUB : SInst<"vhsub", "ddd", "csiUcUsUiQcQsQiQUcQUsQUi">;
263 def VSUBHN : IInst<"vsubhn", "hkk", "silUsUiUl">;
264 def VRSUBHN : IInst<"vrsubhn", "hkk", "silUsUiUl">;
266 ////////////////////////////////////////////////////////////////////////////////
268 def VCEQ : IOpInst<"vceq", "udd", "csifUcUsUiPcQcQsQiQfQUcQUsQUiQPc", OP_EQ>;
269 def VCGE : SOpInst<"vcge", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GE>;
270 let InstName = "vcge" in
271 def VCLE : SOpInst<"vcle", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LE>;
272 def VCGT : SOpInst<"vcgt", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GT>;
273 let InstName = "vcgt" in
274 def VCLT : SOpInst<"vclt", "udd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LT>;
275 let InstName = "vacge" in {
276 def VCAGE : IInst<"vcage", "udd", "fQf">;
277 def VCALE : IInst<"vcale", "udd", "fQf">;
279 let InstName = "vacgt" in {
280 def VCAGT : IInst<"vcagt", "udd", "fQf">;
281 def VCALT : IInst<"vcalt", "udd", "fQf">;
283 def VTST : WInst<"vtst", "udd", "csiUcUsUiPcPsQcQsQiQUcQUsQUiQPcQPs">;
285 ////////////////////////////////////////////////////////////////////////////////
286 // E.3.5 Absolute Difference
287 def VABD : SInst<"vabd", "ddd", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
288 def VABDL : SOpInst<"vabdl", "wdd", "csiUcUsUi", OP_ABDL>;
289 def VABA : SOpInst<"vaba", "dddd", "csiUcUsUiQcQsQiQUcQUsQUi", OP_ABA>;
290 def VABAL : SOpInst<"vabal", "wwdd", "csiUcUsUi", OP_ABAL>;
292 ////////////////////////////////////////////////////////////////////////////////
294 def VMAX : SInst<"vmax", "ddd", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
295 def VMIN : SInst<"vmin", "ddd", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
297 ////////////////////////////////////////////////////////////////////////////////
298 // E.3.7 Pairwise Addition
299 def VPADD : IInst<"vpadd", "ddd", "csiUcUsUif">;
300 def VPADDL : SInst<"vpaddl", "nd", "csiUcUsUiQcQsQiQUcQUsQUi">;
301 def VPADAL : SInst<"vpadal", "nnd", "csiUcUsUiQcQsQiQUcQUsQUi">;
303 ////////////////////////////////////////////////////////////////////////////////
304 // E.3.8-9 Folding Max/Min
305 def VPMAX : SInst<"vpmax", "ddd", "csiUcUsUif">;
306 def VPMIN : SInst<"vpmin", "ddd", "csiUcUsUif">;
308 ////////////////////////////////////////////////////////////////////////////////
309 // E.3.10 Reciprocal/Sqrt
310 def VRECPS : IInst<"vrecps", "ddd", "fQf">;
311 def VRSQRTS : IInst<"vrsqrts", "ddd", "fQf">;
313 ////////////////////////////////////////////////////////////////////////////////
314 // E.3.11 Shifts by signed variable
315 def VSHL : SInst<"vshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
316 def VQSHL : SInst<"vqshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
317 def VRSHL : SInst<"vrshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
318 def VQRSHL : SInst<"vqrshl", "ddx", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
320 ////////////////////////////////////////////////////////////////////////////////
321 // E.3.12 Shifts by constant
323 def VSHR_N : SInst<"vshr_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
324 def VSHL_N : IInst<"vshl_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
325 def VRSHR_N : SInst<"vrshr_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
326 def VSRA_N : SInst<"vsra_n", "dddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
327 def VRSRA_N : SInst<"vrsra_n", "dddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
328 def VQSHL_N : SInst<"vqshl_n", "ddi", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
329 def VQSHLU_N : SInst<"vqshlu_n", "udi", "csilQcQsQiQl">;
330 def VSHRN_N : IInst<"vshrn_n", "hki", "silUsUiUl">;
331 def VQSHRUN_N : SInst<"vqshrun_n", "eki", "sil">;
332 def VQRSHRUN_N : SInst<"vqrshrun_n", "eki", "sil">;
333 def VQSHRN_N : SInst<"vqshrn_n", "hki", "silUsUiUl">;
334 def VRSHRN_N : IInst<"vrshrn_n", "hki", "silUsUiUl">;
335 def VQRSHRN_N : SInst<"vqrshrn_n", "hki", "silUsUiUl">;
336 def VSHLL_N : SInst<"vshll_n", "wdi", "csiUcUsUi">;
338 ////////////////////////////////////////////////////////////////////////////////
339 // E.3.13 Shifts with insert
340 def VSRI_N : WInst<"vsri_n", "dddi",
341 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">;
342 def VSLI_N : WInst<"vsli_n", "dddi",
343 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">;
346 ////////////////////////////////////////////////////////////////////////////////
347 // E.3.14 Loads and stores of a single vector
348 def VLD1 : WInst<"vld1", "dc",
349 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
350 def VLD1_X2 : WInst<"vld1_x2", "2c",
351 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
352 def VLD1_X3 : WInst<"vld1_x3", "3c",
353 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
354 def VLD1_X4 : WInst<"vld1_x4", "4c",
355 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
356 def VLD1_LANE : WInst<"vld1_lane", "dcdi",
357 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
358 def VLD1_DUP : WInst<"vld1_dup", "dc",
359 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
360 def VST1 : WInst<"vst1", "vpd",
361 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
362 def VST1_X2 : WInst<"vst1_x2", "vp2",
363 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
364 def VST1_X3 : WInst<"vst1_x3", "vp3",
365 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
366 def VST1_X4 : WInst<"vst1_x4", "vp4",
367 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
368 def VST1_LANE : WInst<"vst1_lane", "vpdi",
369 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
370 let ArchGuard = "(__ARM_FP & 2)" in {
371 def VLD1_F16 : WInst<"vld1", "dc", "hQh">;
372 def VLD1_X2_F16 : WInst<"vld1_x2", "2c", "hQh">;
373 def VLD1_X3_F16 : WInst<"vld1_x3", "3c", "hQh">;
374 def VLD1_X4_F16 : WInst<"vld1_x4", "4c", "hQh">;
375 def VLD1_LANE_F16 : WInst<"vld1_lane", "dcdi", "hQh">;
376 def VLD1_DUP_F16 : WInst<"vld1_dup", "dc", "hQh">;
377 def VST1_F16 : WInst<"vst1", "vpd", "hQh">;
378 def VST1_X2_F16 : WInst<"vst1_x2", "vp2", "hQh">;
379 def VST1_X3_F16 : WInst<"vst1_x3", "vp3", "hQh">;
380 def VST1_X4_F16 : WInst<"vst1_x4", "vp4", "hQh">;
381 def VST1_LANE_F16 : WInst<"vst1_lane", "vpdi", "hQh">;
384 ////////////////////////////////////////////////////////////////////////////////
385 // E.3.15 Loads and stores of an N-element structure
386 def VLD2 : WInst<"vld2", "2c", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
387 def VLD3 : WInst<"vld3", "3c", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
388 def VLD4 : WInst<"vld4", "4c", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
389 def VLD2_DUP : WInst<"vld2_dup", "2c",
390 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">;
391 def VLD3_DUP : WInst<"vld3_dup", "3c",
392 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">;
393 def VLD4_DUP : WInst<"vld4_dup", "4c",
394 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">;
395 def VLD2_LANE : WInst<"vld2_lane", "2c2i", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
396 def VLD3_LANE : WInst<"vld3_lane", "3c3i", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
397 def VLD4_LANE : WInst<"vld4_lane", "4c4i", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
398 def VST2 : WInst<"vst2", "vp2", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
399 def VST3 : WInst<"vst3", "vp3", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
400 def VST4 : WInst<"vst4", "vp4", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
401 def VST2_LANE : WInst<"vst2_lane", "vp2i", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
402 def VST3_LANE : WInst<"vst3_lane", "vp3i", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
403 def VST4_LANE : WInst<"vst4_lane", "vp4i", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
404 let ArchGuard = "(__ARM_FP & 2)" in {
405 def VLD2_F16 : WInst<"vld2", "2c", "hQh">;
406 def VLD3_F16 : WInst<"vld3", "3c", "hQh">;
407 def VLD4_F16 : WInst<"vld4", "4c", "hQh">;
408 def VLD2_DUP_F16 : WInst<"vld2_dup", "2c", "hQh">;
409 def VLD3_DUP_F16 : WInst<"vld3_dup", "3c", "hQh">;
410 def VLD4_DUP_F16 : WInst<"vld4_dup", "4c", "hQh">;
411 def VLD2_LANE_F16 : WInst<"vld2_lane", "2c2i", "hQh">;
412 def VLD3_LANE_F16 : WInst<"vld3_lane", "3c3i", "hQh">;
413 def VLD4_LANE_F16 : WInst<"vld4_lane", "4c4i", "hQh">;
414 def VST2_F16 : WInst<"vst2", "vp2", "hQh">;
415 def VST3_F16 : WInst<"vst3", "vp3", "hQh">;
416 def VST4_F16 : WInst<"vst4", "vp4", "hQh">;
417 def VST2_LANE_F16 : WInst<"vst2_lane", "vp2i", "hQh">;
418 def VST3_LANE_F16 : WInst<"vst3_lane", "vp3i", "hQh">;
419 def VST4_LANE_F16 : WInst<"vst4_lane", "vp4i", "hQh">;
422 ////////////////////////////////////////////////////////////////////////////////
423 // E.3.16 Extract lanes from a vector
424 let InstName = "vmov" in
425 def VGET_LANE : IInst<"vget_lane", "sdi",
426 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">;
428 ////////////////////////////////////////////////////////////////////////////////
429 // E.3.17 Set lanes within a vector
430 let InstName = "vmov" in
431 def VSET_LANE : IInst<"vset_lane", "dsdi",
432 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">;
434 ////////////////////////////////////////////////////////////////////////////////
435 // E.3.18 Initialize a vector from bit pattern
436 def VCREATE : NoTestOpInst<"vcreate", "dl", "csihfUcUsUiUlPcPsl", OP_CAST> {
437 let BigEndianSafe = 1;
440 ////////////////////////////////////////////////////////////////////////////////
441 // E.3.19 Set all lanes to same value
442 let InstName = "vmov" in {
443 def VDUP_N : WOpInst<"vdup_n", "ds",
444 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl",
446 def VMOV_N : WOpInst<"vmov_n", "ds",
447 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl",
451 def VDUP_LANE: WOpInst<"vdup_lane", "dgi",
452 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl",
455 ////////////////////////////////////////////////////////////////////////////////
456 // E.3.20 Combining vectors
457 def VCOMBINE : NoTestOpInst<"vcombine", "kdd", "csilhfUcUsUiUlPcPs", OP_CONC>;
459 ////////////////////////////////////////////////////////////////////////////////
460 // E.3.21 Splitting vectors
461 // Note that the ARM NEON Reference 2.0 mistakenly document the vget_high_f16()
462 // and vget_low_f16() intrinsics as AArch64-only. We (and GCC) support all
463 // versions of these intrinsics in both AArch32 and AArch64 architectures. See
464 // D45668 for more details.
465 let InstName = "vmov" in {
466 def VGET_HIGH : NoTestOpInst<"vget_high", "dk", "csilhfUcUsUiUlPcPs", OP_HI>;
467 def VGET_LOW : NoTestOpInst<"vget_low", "dk", "csilhfUcUsUiUlPcPs", OP_LO>;
470 ////////////////////////////////////////////////////////////////////////////////
471 // E.3.22 Converting vectors
473 let ArchGuard = "(__ARM_FP & 2)" in {
474 def VCVT_F16_F32 : SInst<"vcvt_f16_f32", "md", "Hf">;
475 def VCVT_F32_F16 : SInst<"vcvt_f32_f16", "wd", "h">;
478 def VCVT_S32 : SInst<"vcvt_s32", "xd", "fQf">;
479 def VCVT_U32 : SInst<"vcvt_u32", "ud", "fQf">;
480 def VCVT_F32 : SInst<"vcvt_f32", "fd", "iUiQiQUi">;
481 let isVCVT_N = 1 in {
482 def VCVT_N_S32 : SInst<"vcvt_n_s32", "xdi", "fQf">;
483 def VCVT_N_U32 : SInst<"vcvt_n_u32", "udi", "fQf">;
484 def VCVT_N_F32 : SInst<"vcvt_n_f32", "fdi", "iUiQiQUi">;
487 def VMOVN : IInst<"vmovn", "hk", "silUsUiUl">;
488 def VMOVL : SInst<"vmovl", "wd", "csiUcUsUi">;
489 def VQMOVN : SInst<"vqmovn", "hk", "silUsUiUl">;
490 def VQMOVUN : SInst<"vqmovun", "ek", "sil">;
492 ////////////////////////////////////////////////////////////////////////////////
493 // E.3.23-24 Table lookup, Extended table lookup
494 let InstName = "vtbl" in {
495 def VTBL1 : WInst<"vtbl1", "ddt", "UccPc">;
496 def VTBL2 : WInst<"vtbl2", "d2t", "UccPc">;
497 def VTBL3 : WInst<"vtbl3", "d3t", "UccPc">;
498 def VTBL4 : WInst<"vtbl4", "d4t", "UccPc">;
500 let InstName = "vtbx" in {
501 def VTBX1 : WInst<"vtbx1", "dddt", "UccPc">;
502 def VTBX2 : WInst<"vtbx2", "dd2t", "UccPc">;
503 def VTBX3 : WInst<"vtbx3", "dd3t", "UccPc">;
504 def VTBX4 : WInst<"vtbx4", "dd4t", "UccPc">;
507 ////////////////////////////////////////////////////////////////////////////////
508 // E.3.25 Operations with a scalar value
509 def VMLA_LANE : IOpInst<"vmla_lane", "dddgi",
510 "siUsUifQsQiQUsQUiQf", OP_MLA_LN>;
511 def VMLAL_LANE : SOpInst<"vmlal_lane", "wwddi", "siUsUi", OP_MLAL_LN>;
512 def VQDMLAL_LANE : SOpInst<"vqdmlal_lane", "wwddi", "si", OP_QDMLAL_LN>;
513 def VMLS_LANE : IOpInst<"vmls_lane", "dddgi",
514 "siUsUifQsQiQUsQUiQf", OP_MLS_LN>;
515 def VMLSL_LANE : SOpInst<"vmlsl_lane", "wwddi", "siUsUi", OP_MLSL_LN>;
516 def VQDMLSL_LANE : SOpInst<"vqdmlsl_lane", "wwddi", "si", OP_QDMLSL_LN>;
517 def VMUL_N : IOpInst<"vmul_n", "dds", "sifUsUiQsQiQfQUsQUi", OP_MUL_N>;
518 def VMUL_LANE : IOpInst<"vmul_lane", "ddgi",
519 "sifUsUiQsQiQfQUsQUi", OP_MUL_LN>;
520 def VMULL_N : SInst<"vmull_n", "wda", "siUsUi">;
521 def VMULL_LANE : SOpInst<"vmull_lane", "wddi", "siUsUi", OP_MULL_LN>;
522 def VQDMULL_N : SInst<"vqdmull_n", "wda", "si">;
523 def VQDMULL_LANE : SOpInst<"vqdmull_lane", "wddi", "si", OP_QDMULL_LN>;
524 def VQDMULH_N : SInst<"vqdmulh_n", "dda", "siQsQi">;
525 def VQDMULH_LANE : SOpInst<"vqdmulh_lane", "ddgi", "siQsQi", OP_QDMULH_LN>;
526 def VQRDMULH_N : SInst<"vqrdmulh_n", "dda", "siQsQi">;
527 def VQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "ddgi", "siQsQi", OP_QRDMULH_LN>;
529 let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in {
530 def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "dddgi", "siQsQi", OP_QRDMLAH_LN>;
531 def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "dddgi", "siQsQi", OP_QRDMLSH_LN>;
534 def VMLA_N : IOpInst<"vmla_n", "ddda", "siUsUifQsQiQUsQUiQf", OP_MLA_N>;
535 def VMLAL_N : SOpInst<"vmlal_n", "wwda", "siUsUi", OP_MLAL_N>;
536 def VQDMLAL_N : SInst<"vqdmlal_n", "wwda", "si">;
537 def VMLS_N : IOpInst<"vmls_n", "ddds", "siUsUifQsQiQUsQUiQf", OP_MLS_N>;
538 def VMLSL_N : SOpInst<"vmlsl_n", "wwda", "siUsUi", OP_MLSL_N>;
539 def VQDMLSL_N : SInst<"vqdmlsl_n", "wwda", "si">;
541 ////////////////////////////////////////////////////////////////////////////////
542 // E.3.26 Vector Extract
543 def VEXT : WInst<"vext", "dddi",
544 "cUcPcsUsPsiUilUlfQcQUcQPcQsQUsQPsQiQUiQlQUlQf">;
546 ////////////////////////////////////////////////////////////////////////////////
547 // E.3.27 Reverse vector elements
548 def VREV64 : WOpInst<"vrev64", "dd", "csiUcUsUiPcPsfQcQsQiQUcQUsQUiQPcQPsQf",
550 def VREV32 : WOpInst<"vrev32", "dd", "csUcUsPcPsQcQsQUcQUsQPcQPs", OP_REV32>;
551 def VREV16 : WOpInst<"vrev16", "dd", "cUcPcQcQUcQPc", OP_REV16>;
553 ////////////////////////////////////////////////////////////////////////////////
554 // E.3.28 Other single operand arithmetic
555 def VABS : SInst<"vabs", "dd", "csifQcQsQiQf">;
556 def VQABS : SInst<"vqabs", "dd", "csiQcQsQi">;
557 def VNEG : SOpInst<"vneg", "dd", "csifQcQsQiQf", OP_NEG>;
558 def VQNEG : SInst<"vqneg", "dd", "csiQcQsQi">;
559 def VCLS : SInst<"vcls", "dd", "csiQcQsQi">;
560 def VCLZ : IInst<"vclz", "dd", "csiUcUsUiQcQsQiQUcQUsQUi">;
561 def VCNT : WInst<"vcnt", "dd", "UccPcQUcQcQPc">;
562 def VRECPE : SInst<"vrecpe", "dd", "fUiQfQUi">;
563 def VRSQRTE : SInst<"vrsqrte", "dd", "fUiQfQUi">;
565 ////////////////////////////////////////////////////////////////////////////////
566 // E.3.29 Logical operations
567 def VMVN : LOpInst<"vmvn", "dd", "csiUcUsUiPcQcQsQiQUcQUsQUiQPc", OP_NOT>;
568 def VAND : LOpInst<"vand", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_AND>;
569 def VORR : LOpInst<"vorr", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_OR>;
570 def VEOR : LOpInst<"veor", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_XOR>;
571 def VBIC : LOpInst<"vbic", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ANDN>;
572 def VORN : LOpInst<"vorn", "ddd", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ORN>;
573 let isHiddenLInst = 1 in
574 def VBSL : SInst<"vbsl", "dudd",
575 "csilUcUsUiUlfPcPsQcQsQiQlQUcQUsQUiQUlQfQPcQPs">;
577 ////////////////////////////////////////////////////////////////////////////////
578 // E.3.30 Transposition operations
579 def VTRN : WInst<"vtrn", "2dd", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
580 def VZIP : WInst<"vzip", "2dd", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
581 def VUZP : WInst<"vuzp", "2dd", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
583 ////////////////////////////////////////////////////////////////////////////////
584 // E.3.31 Vector reinterpret cast operations
586 : NoTestOpInst<"vreinterpret", "dd",
587 "csilUcUsUiUlhfPcPsQcQsQiQlQUcQUsQUiQUlQhQfQPcQPs", OP_REINT> {
588 let CartesianProductOfTypes = 1;
589 let ArchGuard = "!defined(__aarch64__)";
590 let BigEndianSafe = 1;
593 ////////////////////////////////////////////////////////////////////////////////
594 // Vector fused multiply-add operations
596 let ArchGuard = "defined(__ARM_FEATURE_FMA)" in {
597 def VFMA : SInst<"vfma", "dddd", "fQf">;
598 def VFMS : SOpInst<"vfms", "dddd", "fQf", OP_FMLS>;
599 def FMLA_N_F32 : SOpInst<"vfma_n", "ddds", "fQf", OP_FMLA_N>;
602 ////////////////////////////////////////////////////////////////////////////////
603 // fp16 vector operations
604 def SCALAR_HALF_GET_LANE : IOpInst<"vget_lane", "sdi", "h", OP_SCALAR_HALF_GET_LN>;
605 def SCALAR_HALF_SET_LANE : IOpInst<"vset_lane", "dsdi", "h", OP_SCALAR_HALF_SET_LN>;
606 def SCALAR_HALF_GET_LANEQ : IOpInst<"vget_lane", "sdi", "Qh", OP_SCALAR_HALF_GET_LNQ>;
607 def SCALAR_HALF_SET_LANEQ : IOpInst<"vset_lane", "dsdi", "Qh", OP_SCALAR_HALF_SET_LNQ>;
609 ////////////////////////////////////////////////////////////////////////////////
610 // AArch64 Intrinsics
612 let ArchGuard = "defined(__aarch64__)" in {
614 ////////////////////////////////////////////////////////////////////////////////
616 def LD1 : WInst<"vld1", "dc", "dQdPlQPl">;
617 def LD2 : WInst<"vld2", "2c", "QUlQldQdPlQPl">;
618 def LD3 : WInst<"vld3", "3c", "QUlQldQdPlQPl">;
619 def LD4 : WInst<"vld4", "4c", "QUlQldQdPlQPl">;
620 def ST1 : WInst<"vst1", "vpd", "dQdPlQPl">;
621 def ST2 : WInst<"vst2", "vp2", "QUlQldQdPlQPl">;
622 def ST3 : WInst<"vst3", "vp3", "QUlQldQdPlQPl">;
623 def ST4 : WInst<"vst4", "vp4", "QUlQldQdPlQPl">;
625 def LD1_X2 : WInst<"vld1_x2", "2c",
627 def LD1_X3 : WInst<"vld1_x3", "3c",
629 def LD1_X4 : WInst<"vld1_x4", "4c",
632 def ST1_X2 : WInst<"vst1_x2", "vp2", "dQdPlQPl">;
633 def ST1_X3 : WInst<"vst1_x3", "vp3", "dQdPlQPl">;
634 def ST1_X4 : WInst<"vst1_x4", "vp4", "dQdPlQPl">;
636 def LD1_LANE : WInst<"vld1_lane", "dcdi", "dQdPlQPl">;
637 def LD2_LANE : WInst<"vld2_lane", "2c2i", "lUlQcQUcQPcQlQUldQdPlQPl">;
638 def LD3_LANE : WInst<"vld3_lane", "3c3i", "lUlQcQUcQPcQlQUldQdPlQPl">;
639 def LD4_LANE : WInst<"vld4_lane", "4c4i", "lUlQcQUcQPcQlQUldQdPlQPl">;
640 def ST1_LANE : WInst<"vst1_lane", "vpdi", "dQdPlQPl">;
641 def ST2_LANE : WInst<"vst2_lane", "vp2i", "lUlQcQUcQPcQlQUldQdPlQPl">;
642 def ST3_LANE : WInst<"vst3_lane", "vp3i", "lUlQcQUcQPcQlQUldQdPlQPl">;
643 def ST4_LANE : WInst<"vst4_lane", "vp4i", "lUlQcQUcQPcQlQUldQdPlQPl">;
645 def LD1_DUP : WInst<"vld1_dup", "dc", "dQdPlQPl">;
646 def LD2_DUP : WInst<"vld2_dup", "2c", "dQdPlQPl">;
647 def LD3_DUP : WInst<"vld3_dup", "3c", "dQdPlQPl">;
648 def LD4_DUP : WInst<"vld4_dup", "4c", "dQdPlQPl">;
650 def VLDRQ : WInst<"vldrq", "sc", "Pk">;
651 def VSTRQ : WInst<"vstrq", "vps", "Pk">;
653 ////////////////////////////////////////////////////////////////////////////////
655 def ADD : IOpInst<"vadd", "ddd", "dQd", OP_ADD>;
657 ////////////////////////////////////////////////////////////////////////////////
659 def SUB : IOpInst<"vsub", "ddd", "dQd", OP_SUB>;
661 ////////////////////////////////////////////////////////////////////////////////
663 def MUL : IOpInst<"vmul", "ddd", "dQd", OP_MUL>;
664 def MLA : IOpInst<"vmla", "dddd", "dQd", OP_MLA>;
665 def MLS : IOpInst<"vmls", "dddd", "dQd", OP_MLS>;
667 ////////////////////////////////////////////////////////////////////////////////
668 // Multiplication Extended
669 def MULX : SInst<"vmulx", "ddd", "fdQfQd">;
671 ////////////////////////////////////////////////////////////////////////////////
673 def FDIV : IOpInst<"vdiv", "ddd", "fdQfQd", OP_DIV>;
675 ////////////////////////////////////////////////////////////////////////////////
676 // Vector fused multiply-add operations
677 def FMLA : SInst<"vfma", "dddd", "dQd">;
678 def FMLS : SOpInst<"vfms", "dddd", "dQd", OP_FMLS>;
680 ////////////////////////////////////////////////////////////////////////////////
681 // MUL, MLA, MLS, FMA, FMS definitions with scalar argument
682 def VMUL_N_A64 : IOpInst<"vmul_n", "dds", "Qd", OP_MUL_N>;
684 def FMLA_N : SOpInst<"vfma_n", "ddds", "dQd", OP_FMLA_N>;
685 def FMLS_N : SOpInst<"vfms_n", "ddds", "fdQfQd", OP_FMLS_N>;
687 def MLA_N : SOpInst<"vmla_n", "ddds", "Qd", OP_MLA_N>;
688 def MLS_N : SOpInst<"vmls_n", "ddds", "Qd", OP_MLS_N>;
690 ////////////////////////////////////////////////////////////////////////////////
691 // Logical operations
692 def BSL : SInst<"vbsl", "dudd", "dPlQdQPl">;
694 ////////////////////////////////////////////////////////////////////////////////
695 // Absolute Difference
696 def ABD : SInst<"vabd", "ddd", "dQd">;
698 ////////////////////////////////////////////////////////////////////////////////
699 // saturating absolute/negate
700 def ABS : SInst<"vabs", "dd", "dQdlQl">;
701 def QABS : SInst<"vqabs", "dd", "lQl">;
702 def NEG : SOpInst<"vneg", "dd", "dlQdQl", OP_NEG>;
703 def QNEG : SInst<"vqneg", "dd", "lQl">;
705 ////////////////////////////////////////////////////////////////////////////////
706 // Signed Saturating Accumulated of Unsigned Value
707 def SUQADD : SInst<"vuqadd", "ddd", "csilQcQsQiQl">;
709 ////////////////////////////////////////////////////////////////////////////////
710 // Unsigned Saturating Accumulated of Signed Value
711 def USQADD : SInst<"vsqadd", "ddd", "UcUsUiUlQUcQUsQUiQUl">;
713 ////////////////////////////////////////////////////////////////////////////////
715 def FRECPS : IInst<"vrecps", "ddd", "dQd">;
716 def FRSQRTS : IInst<"vrsqrts", "ddd", "dQd">;
717 def FRECPE : SInst<"vrecpe", "dd", "dQd">;
718 def FRSQRTE : SInst<"vrsqrte", "dd", "dQd">;
719 def FSQRT : SInst<"vsqrt", "dd", "fdQfQd">;
721 ////////////////////////////////////////////////////////////////////////////////
723 def RBIT : IInst<"vrbit", "dd", "cUcPcQcQUcQPc">;
725 ////////////////////////////////////////////////////////////////////////////////
726 // Integer extract and narrow to high
727 def XTN2 : SOpInst<"vmovn_high", "qhk", "silUsUiUl", OP_XTN>;
729 ////////////////////////////////////////////////////////////////////////////////
730 // Signed integer saturating extract and unsigned narrow to high
731 def SQXTUN2 : SOpInst<"vqmovun_high", "emd", "HsHiHl", OP_SQXTUN>;
733 ////////////////////////////////////////////////////////////////////////////////
734 // Integer saturating extract and narrow to high
735 def QXTN2 : SOpInst<"vqmovn_high", "qhk", "silUsUiUl", OP_QXTN>;
737 ////////////////////////////////////////////////////////////////////////////////
738 // Converting vectors
740 def VCVT_F32_F64 : SInst<"vcvt_f32_f64", "md", "Qd">;
741 def VCVT_F64_F32 : SInst<"vcvt_f64_f32", "wd", "f">;
743 def VCVT_S64 : SInst<"vcvt_s64", "xd", "dQd">;
744 def VCVT_U64 : SInst<"vcvt_u64", "ud", "dQd">;
745 def VCVT_F64 : SInst<"vcvt_f64", "Fd", "lUlQlQUl">;
747 def VCVT_HIGH_F16_F32 : SOpInst<"vcvt_high_f16", "hmj", "Hf", OP_VCVT_NA_HI_F16>;
748 def VCVT_HIGH_F32_F16 : SOpInst<"vcvt_high_f32", "wk", "h", OP_VCVT_EX_HI_F32>;
749 def VCVT_HIGH_F32_F64 : SOpInst<"vcvt_high_f32", "qfj", "d", OP_VCVT_NA_HI_F32>;
750 def VCVT_HIGH_F64_F32 : SOpInst<"vcvt_high_f64", "wj", "f", OP_VCVT_EX_HI_F64>;
752 def VCVTX_F32_F64 : SInst<"vcvtx_f32", "fj", "d">;
753 def VCVTX_HIGH_F32_F64 : SOpInst<"vcvtx_high_f32", "qfj", "d", OP_VCVTX_HI>;
755 ////////////////////////////////////////////////////////////////////////////////
757 def FCAGE : IInst<"vcage", "udd", "dQd">;
758 def FCAGT : IInst<"vcagt", "udd", "dQd">;
759 def FCALE : IInst<"vcale", "udd", "dQd">;
760 def FCALT : IInst<"vcalt", "udd", "dQd">;
761 def CMTST : WInst<"vtst", "udd", "lUlPlQlQUlQPl">;
762 def CFMEQ : SOpInst<"vceq", "udd", "lUldQdQlQUlPlQPl", OP_EQ>;
763 def CFMGE : SOpInst<"vcge", "udd", "lUldQdQlQUl", OP_GE>;
764 def CFMLE : SOpInst<"vcle", "udd", "lUldQdQlQUl", OP_LE>;
765 def CFMGT : SOpInst<"vcgt", "udd", "lUldQdQlQUl", OP_GT>;
766 def CFMLT : SOpInst<"vclt", "udd", "lUldQdQlQUl", OP_LT>;
768 def CMEQ : SInst<"vceqz", "ud",
769 "csilfUcUsUiUlPcPsPlQcQsQiQlQfQUcQUsQUiQUlQPcQPsdQdQPl">;
770 def CMGE : SInst<"vcgez", "ud", "csilfdQcQsQiQlQfQd">;
771 def CMLE : SInst<"vclez", "ud", "csilfdQcQsQiQlQfQd">;
772 def CMGT : SInst<"vcgtz", "ud", "csilfdQcQsQiQlQfQd">;
773 def CMLT : SInst<"vcltz", "ud", "csilfdQcQsQiQlQfQd">;
775 ////////////////////////////////////////////////////////////////////////////////
777 def MAX : SInst<"vmax", "ddd", "dQd">;
778 def MIN : SInst<"vmin", "ddd", "dQd">;
780 ////////////////////////////////////////////////////////////////////////////////
782 def MAXP : SInst<"vpmax", "ddd", "QcQsQiQUcQUsQUiQfQd">;
783 def MINP : SInst<"vpmin", "ddd", "QcQsQiQUcQUsQUiQfQd">;
785 ////////////////////////////////////////////////////////////////////////////////
786 // Pairwise MaxNum/MinNum Floating Point
787 def FMAXNMP : SInst<"vpmaxnm", "ddd", "fQfQd">;
788 def FMINNMP : SInst<"vpminnm", "ddd", "fQfQd">;
790 ////////////////////////////////////////////////////////////////////////////////
792 def ADDP : IInst<"vpadd", "ddd", "QcQsQiQlQUcQUsQUiQUlQfQd">;
794 ////////////////////////////////////////////////////////////////////////////////
795 // Shifts by constant
797 // Left shift long high
798 def SHLL_HIGH_N : SOpInst<"vshll_high_n", "ndi", "HcHsHiHUcHUsHUi",
801 ////////////////////////////////////////////////////////////////////////////////
802 def SRI_N : WInst<"vsri_n", "dddi", "PlQPl">;
803 def SLI_N : WInst<"vsli_n", "dddi", "PlQPl">;
805 // Right shift narrow high
806 def SHRN_HIGH_N : IOpInst<"vshrn_high_n", "hmdi",
807 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
808 def QSHRUN_HIGH_N : SOpInst<"vqshrun_high_n", "hmdi",
809 "HsHiHl", OP_NARROW_HI>;
810 def RSHRN_HIGH_N : IOpInst<"vrshrn_high_n", "hmdi",
811 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
812 def QRSHRUN_HIGH_N : SOpInst<"vqrshrun_high_n", "hmdi",
813 "HsHiHl", OP_NARROW_HI>;
814 def QSHRN_HIGH_N : SOpInst<"vqshrn_high_n", "hmdi",
815 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
816 def QRSHRN_HIGH_N : SOpInst<"vqrshrn_high_n", "hmdi",
817 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
820 ////////////////////////////////////////////////////////////////////////////////
821 // Converting vectors
822 def VMOVL_HIGH : SOpInst<"vmovl_high", "nd", "HcHsHiHUcHUsHUi", OP_MOVL_HI>;
824 let isVCVT_N = 1 in {
825 def CVTF_N_F64 : SInst<"vcvt_n_f64", "Fdi", "lUlQlQUl">;
826 def FCVTZS_N_S64 : SInst<"vcvt_n_s64", "xdi", "dQd">;
827 def FCVTZS_N_U64 : SInst<"vcvt_n_u64", "udi", "dQd">;
830 ////////////////////////////////////////////////////////////////////////////////
831 // 3VDiff class using high 64-bit in operands
832 def VADDL_HIGH : SOpInst<"vaddl_high", "wkk", "csiUcUsUi", OP_ADDLHi>;
833 def VADDW_HIGH : SOpInst<"vaddw_high", "wwk", "csiUcUsUi", OP_ADDWHi>;
834 def VSUBL_HIGH : SOpInst<"vsubl_high", "wkk", "csiUcUsUi", OP_SUBLHi>;
835 def VSUBW_HIGH : SOpInst<"vsubw_high", "wwk", "csiUcUsUi", OP_SUBWHi>;
837 def VABDL_HIGH : SOpInst<"vabdl_high", "wkk", "csiUcUsUi", OP_ABDLHi>;
838 def VABAL_HIGH : SOpInst<"vabal_high", "wwkk", "csiUcUsUi", OP_ABALHi>;
840 def VMULL_HIGH : SOpInst<"vmull_high", "wkk", "csiUcUsUiPc", OP_MULLHi>;
841 def VMULL_HIGH_N : SOpInst<"vmull_high_n", "wks", "siUsUi", OP_MULLHi_N>;
842 def VMLAL_HIGH : SOpInst<"vmlal_high", "wwkk", "csiUcUsUi", OP_MLALHi>;
843 def VMLAL_HIGH_N : SOpInst<"vmlal_high_n", "wwks", "siUsUi", OP_MLALHi_N>;
844 def VMLSL_HIGH : SOpInst<"vmlsl_high", "wwkk", "csiUcUsUi", OP_MLSLHi>;
845 def VMLSL_HIGH_N : SOpInst<"vmlsl_high_n", "wwks", "siUsUi", OP_MLSLHi_N>;
847 def VADDHN_HIGH : SOpInst<"vaddhn_high", "qhkk", "silUsUiUl", OP_ADDHNHi>;
848 def VRADDHN_HIGH : SOpInst<"vraddhn_high", "qhkk", "silUsUiUl", OP_RADDHNHi>;
849 def VSUBHN_HIGH : SOpInst<"vsubhn_high", "qhkk", "silUsUiUl", OP_SUBHNHi>;
850 def VRSUBHN_HIGH : SOpInst<"vrsubhn_high", "qhkk", "silUsUiUl", OP_RSUBHNHi>;
852 def VQDMULL_HIGH : SOpInst<"vqdmull_high", "wkk", "si", OP_QDMULLHi>;
853 def VQDMULL_HIGH_N : SOpInst<"vqdmull_high_n", "wks", "si", OP_QDMULLHi_N>;
854 def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "wwkk", "si", OP_QDMLALHi>;
855 def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "wwks", "si", OP_QDMLALHi_N>;
856 def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "wwkk", "si", OP_QDMLSLHi>;
857 def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "wwks", "si", OP_QDMLSLHi_N>;
858 def VMULL_P64 : SInst<"vmull", "rss", "Pl">;
859 def VMULL_HIGH_P64 : SOpInst<"vmull_high", "rdd", "HPl", OP_MULLHi_P64>;
862 ////////////////////////////////////////////////////////////////////////////////
863 // Extract or insert element from vector
864 def GET_LANE : IInst<"vget_lane", "sdi", "dQdPlQPl">;
865 def SET_LANE : IInst<"vset_lane", "dsdi", "dQdPlQPl">;
866 def COPY_LANE : IOpInst<"vcopy_lane", "ddidi",
867 "csilUcUsUiUlPcPsPlfd", OP_COPY_LN>;
868 def COPYQ_LANE : IOpInst<"vcopy_lane", "ddigi",
869 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN>;
870 def COPY_LANEQ : IOpInst<"vcopy_laneq", "ddiki",
871 "csilPcPsPlUcUsUiUlfd", OP_COPY_LN>;
872 def COPYQ_LANEQ : IOpInst<"vcopy_laneq", "ddidi",
873 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN>;
875 ////////////////////////////////////////////////////////////////////////////////
876 // Set all lanes to same value
877 def VDUP_LANE1: WOpInst<"vdup_lane", "dgi", "hdQhQdPlQPl", OP_DUP_LN>;
878 def VDUP_LANE2: WOpInst<"vdup_laneq", "dji",
879 "csilUcUsUiUlPcPshfdQcQsQiQlQPcQPsQUcQUsQUiQUlQhQfQdPlQPl",
881 def DUP_N : WOpInst<"vdup_n", "ds", "dQdPlQPl", OP_DUP>;
882 def MOV_N : WOpInst<"vmov_n", "ds", "dQdPlQPl", OP_DUP>;
884 ////////////////////////////////////////////////////////////////////////////////
885 def COMBINE : NoTestOpInst<"vcombine", "kdd", "dPl", OP_CONC>;
887 ////////////////////////////////////////////////////////////////////////////////
888 //Initialize a vector from bit pattern
889 def CREATE : NoTestOpInst<"vcreate", "dl", "dPl", OP_CAST> {
890 let BigEndianSafe = 1;
893 ////////////////////////////////////////////////////////////////////////////////
895 def VMLA_LANEQ : IOpInst<"vmla_laneq", "dddji",
896 "siUsUifQsQiQUsQUiQf", OP_MLA_LN>;
897 def VMLS_LANEQ : IOpInst<"vmls_laneq", "dddji",
898 "siUsUifQsQiQUsQUiQf", OP_MLS_LN>;
900 def VFMA_LANE : IInst<"vfma_lane", "dddgi", "fdQfQd">;
901 def VFMA_LANEQ : IInst<"vfma_laneq", "dddji", "fdQfQd"> {
904 def VFMS_LANE : IOpInst<"vfms_lane", "dddgi", "fdQfQd", OP_FMS_LN>;
905 def VFMS_LANEQ : IOpInst<"vfms_laneq", "dddji", "fdQfQd", OP_FMS_LNQ>;
907 def VMLAL_LANEQ : SOpInst<"vmlal_laneq", "wwdki", "siUsUi", OP_MLAL_LN>;
908 def VMLAL_HIGH_LANE : SOpInst<"vmlal_high_lane", "wwkdi", "siUsUi",
910 def VMLAL_HIGH_LANEQ : SOpInst<"vmlal_high_laneq", "wwkki", "siUsUi",
912 def VMLSL_LANEQ : SOpInst<"vmlsl_laneq", "wwdki", "siUsUi", OP_MLSL_LN>;
913 def VMLSL_HIGH_LANE : SOpInst<"vmlsl_high_lane", "wwkdi", "siUsUi",
915 def VMLSL_HIGH_LANEQ : SOpInst<"vmlsl_high_laneq", "wwkki", "siUsUi",
918 def VQDMLAL_LANEQ : SOpInst<"vqdmlal_laneq", "wwdki", "si", OP_QDMLAL_LN>;
919 def VQDMLAL_HIGH_LANE : SOpInst<"vqdmlal_high_lane", "wwkdi", "si",
921 def VQDMLAL_HIGH_LANEQ : SOpInst<"vqdmlal_high_laneq", "wwkki", "si",
923 def VQDMLSL_LANEQ : SOpInst<"vqdmlsl_laneq", "wwdki", "si", OP_QDMLSL_LN>;
924 def VQDMLSL_HIGH_LANE : SOpInst<"vqdmlsl_high_lane", "wwkdi", "si",
926 def VQDMLSL_HIGH_LANEQ : SOpInst<"vqdmlsl_high_laneq", "wwkki", "si",
929 // Newly add double parameter for vmul_lane in aarch64
930 // Note: d type is handled by SCALAR_VMUL_LANE
931 def VMUL_LANE_A64 : IOpInst<"vmul_lane", "ddgi", "Qd", OP_MUL_LN>;
933 // Note: d type is handled by SCALAR_VMUL_LANEQ
934 def VMUL_LANEQ : IOpInst<"vmul_laneq", "ddji",
935 "sifUsUiQsQiQUsQUiQfQd", OP_MUL_LN>;
936 def VMULL_LANEQ : SOpInst<"vmull_laneq", "wdki", "siUsUi", OP_MULL_LN>;
937 def VMULL_HIGH_LANE : SOpInst<"vmull_high_lane", "wkdi", "siUsUi",
939 def VMULL_HIGH_LANEQ : SOpInst<"vmull_high_laneq", "wkki", "siUsUi",
942 def VQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "wdki", "si", OP_QDMULL_LN>;
943 def VQDMULL_HIGH_LANE : SOpInst<"vqdmull_high_lane", "wkdi", "si",
945 def VQDMULL_HIGH_LANEQ : SOpInst<"vqdmull_high_laneq", "wkki", "si",
948 def VQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "ddji", "siQsQi", OP_QDMULH_LN>;
949 def VQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "ddji", "siQsQi", OP_QRDMULH_LN>;
951 let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
952 def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "dddji", "siQsQi", OP_QRDMLAH_LN>;
953 def VQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "dddji", "siQsQi", OP_QRDMLSH_LN>;
956 // Note: d type implemented by SCALAR_VMULX_LANE
957 def VMULX_LANE : IOpInst<"vmulx_lane", "ddgi", "fQfQd", OP_MULX_LN>;
958 // Note: d type is implemented by SCALAR_VMULX_LANEQ
959 def VMULX_LANEQ : IOpInst<"vmulx_laneq", "ddji", "fQfQd", OP_MULX_LN>;
961 ////////////////////////////////////////////////////////////////////////////////
962 // Across vectors class
963 def VADDLV : SInst<"vaddlv", "rd", "csiUcUsUiQcQsQiQUcQUsQUi">;
964 def VMAXV : SInst<"vmaxv", "sd", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">;
965 def VMINV : SInst<"vminv", "sd", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">;
966 def VADDV : SInst<"vaddv", "sd", "csifUcUsUiQcQsQiQUcQUsQUiQfQdQlQUl">;
967 def FMAXNMV : SInst<"vmaxnmv", "sd", "fQfQd">;
968 def FMINNMV : SInst<"vminnmv", "sd", "fQfQd">;
970 ////////////////////////////////////////////////////////////////////////////////
971 // Newly added Vector Extract for f64
972 def VEXT_A64 : WInst<"vext", "dddi", "dQdPlQPl">;
974 ////////////////////////////////////////////////////////////////////////////////
976 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_CRYPTO)" in {
977 def AESE : SInst<"vaese", "ddd", "QUc">;
978 def AESD : SInst<"vaesd", "ddd", "QUc">;
979 def AESMC : SInst<"vaesmc", "dd", "QUc">;
980 def AESIMC : SInst<"vaesimc", "dd", "QUc">;
982 def SHA1H : SInst<"vsha1h", "ss", "Ui">;
983 def SHA1SU1 : SInst<"vsha1su1", "ddd", "QUi">;
984 def SHA256SU0 : SInst<"vsha256su0", "ddd", "QUi">;
986 def SHA1C : SInst<"vsha1c", "ddsd", "QUi">;
987 def SHA1P : SInst<"vsha1p", "ddsd", "QUi">;
988 def SHA1M : SInst<"vsha1m", "ddsd", "QUi">;
989 def SHA1SU0 : SInst<"vsha1su0", "dddd", "QUi">;
990 def SHA256H : SInst<"vsha256h", "dddd", "QUi">;
991 def SHA256H2 : SInst<"vsha256h2", "dddd", "QUi">;
992 def SHA256SU1 : SInst<"vsha256su1", "dddd", "QUi">;
995 ////////////////////////////////////////////////////////////////////////////////
996 // Float -> Int conversions with explicit rounding mode
998 let ArchGuard = "__ARM_ARCH >= 8" in {
999 def FCVTNS_S32 : SInst<"vcvtn_s32", "xd", "fQf">;
1000 def FCVTNU_S32 : SInst<"vcvtn_u32", "ud", "fQf">;
1001 def FCVTPS_S32 : SInst<"vcvtp_s32", "xd", "fQf">;
1002 def FCVTPU_S32 : SInst<"vcvtp_u32", "ud", "fQf">;
1003 def FCVTMS_S32 : SInst<"vcvtm_s32", "xd", "fQf">;
1004 def FCVTMU_S32 : SInst<"vcvtm_u32", "ud", "fQf">;
1005 def FCVTAS_S32 : SInst<"vcvta_s32", "xd", "fQf">;
1006 def FCVTAU_S32 : SInst<"vcvta_u32", "ud", "fQf">;
1009 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)" in {
1010 def FCVTNS_S64 : SInst<"vcvtn_s64", "xd", "dQd">;
1011 def FCVTNU_S64 : SInst<"vcvtn_u64", "ud", "dQd">;
1012 def FCVTPS_S64 : SInst<"vcvtp_s64", "xd", "dQd">;
1013 def FCVTPU_S64 : SInst<"vcvtp_u64", "ud", "dQd">;
1014 def FCVTMS_S64 : SInst<"vcvtm_s64", "xd", "dQd">;
1015 def FCVTMU_S64 : SInst<"vcvtm_u64", "ud", "dQd">;
1016 def FCVTAS_S64 : SInst<"vcvta_s64", "xd", "dQd">;
1017 def FCVTAU_S64 : SInst<"vcvta_u64", "ud", "dQd">;
1020 ////////////////////////////////////////////////////////////////////////////////
1021 // Round to Integral
1023 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1024 def FRINTN_S32 : SInst<"vrndn", "dd", "fQf">;
1025 def FRINTA_S32 : SInst<"vrnda", "dd", "fQf">;
1026 def FRINTP_S32 : SInst<"vrndp", "dd", "fQf">;
1027 def FRINTM_S32 : SInst<"vrndm", "dd", "fQf">;
1028 def FRINTX_S32 : SInst<"vrndx", "dd", "fQf">;
1029 def FRINTZ_S32 : SInst<"vrnd", "dd", "fQf">;
1030 def FRINTI_S32 : SInst<"vrndi", "dd", "fQf">;
1033 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1034 def FRINTN_S64 : SInst<"vrndn", "dd", "dQd">;
1035 def FRINTA_S64 : SInst<"vrnda", "dd", "dQd">;
1036 def FRINTP_S64 : SInst<"vrndp", "dd", "dQd">;
1037 def FRINTM_S64 : SInst<"vrndm", "dd", "dQd">;
1038 def FRINTX_S64 : SInst<"vrndx", "dd", "dQd">;
1039 def FRINTZ_S64 : SInst<"vrnd", "dd", "dQd">;
1040 def FRINTI_S64 : SInst<"vrndi", "dd", "dQd">;
1043 ////////////////////////////////////////////////////////////////////////////////
1044 // MaxNum/MinNum Floating Point
1046 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in {
1047 def FMAXNM_S32 : SInst<"vmaxnm", "ddd", "fQf">;
1048 def FMINNM_S32 : SInst<"vminnm", "ddd", "fQf">;
1051 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in {
1052 def FMAXNM_S64 : SInst<"vmaxnm", "ddd", "dQd">;
1053 def FMINNM_S64 : SInst<"vminnm", "ddd", "dQd">;
1056 ////////////////////////////////////////////////////////////////////////////////
1058 def VTRN1 : SOpInst<"vtrn1", "ddd",
1059 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN1>;
1060 def VZIP1 : SOpInst<"vzip1", "ddd",
1061 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP1>;
1062 def VUZP1 : SOpInst<"vuzp1", "ddd",
1063 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP1>;
1064 def VTRN2 : SOpInst<"vtrn2", "ddd",
1065 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN2>;
1066 def VZIP2 : SOpInst<"vzip2", "ddd",
1067 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP2>;
1068 def VUZP2 : SOpInst<"vuzp2", "ddd",
1069 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP2>;
1071 ////////////////////////////////////////////////////////////////////////////////
1073 let InstName = "vtbl" in {
1074 def VQTBL1_A64 : WInst<"vqtbl1", "djt", "UccPcQUcQcQPc">;
1075 def VQTBL2_A64 : WInst<"vqtbl2", "dBt", "UccPcQUcQcQPc">;
1076 def VQTBL3_A64 : WInst<"vqtbl3", "dCt", "UccPcQUcQcQPc">;
1077 def VQTBL4_A64 : WInst<"vqtbl4", "dDt", "UccPcQUcQcQPc">;
1079 let InstName = "vtbx" in {
1080 def VQTBX1_A64 : WInst<"vqtbx1", "ddjt", "UccPcQUcQcQPc">;
1081 def VQTBX2_A64 : WInst<"vqtbx2", "ddBt", "UccPcQUcQcQPc">;
1082 def VQTBX3_A64 : WInst<"vqtbx3", "ddCt", "UccPcQUcQcQPc">;
1083 def VQTBX4_A64 : WInst<"vqtbx4", "ddDt", "UccPcQUcQcQPc">;
1086 ////////////////////////////////////////////////////////////////////////////////
1087 // Vector reinterpret cast operations
1089 // NeonEmitter implicitly takes the cartesian product of the type string with
1090 // itself during generation so, unlike all other intrinsics, this one should
1091 // include *all* types, not just additional ones.
1093 : NoTestOpInst<"vreinterpret", "dd",
1094 "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", OP_REINT> {
1095 let CartesianProductOfTypes = 1;
1096 let BigEndianSafe = 1;
1097 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)";
1100 ////////////////////////////////////////////////////////////////////////////////
1101 // Scalar Intrinsics
1102 // Scalar Arithmetic
1105 def SCALAR_ADD : SInst<"vadd", "sss", "SlSUl">;
1106 // Scalar Saturating Add
1107 def SCALAR_QADD : SInst<"vqadd", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1109 // Scalar Subtraction
1110 def SCALAR_SUB : SInst<"vsub", "sss", "SlSUl">;
1111 // Scalar Saturating Sub
1112 def SCALAR_QSUB : SInst<"vqsub", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1114 let InstName = "vmov" in {
1115 def VGET_HIGH_A64 : NoTestOpInst<"vget_high", "dk", "dPl", OP_HI>;
1116 def VGET_LOW_A64 : NoTestOpInst<"vget_low", "dk", "dPl", OP_LO>;
1119 ////////////////////////////////////////////////////////////////////////////////
1121 // Scalar Shift Left
1122 def SCALAR_SHL: SInst<"vshl", "sss", "SlSUl">;
1123 // Scalar Saturating Shift Left
1124 def SCALAR_QSHL: SInst<"vqshl", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1125 // Scalar Saturating Rounding Shift Left
1126 def SCALAR_QRSHL: SInst<"vqrshl", "sss", "ScSsSiSlSUcSUsSUiSUl">;
1127 // Scalar Shift Rounding Left
1128 def SCALAR_RSHL: SInst<"vrshl", "sss", "SlSUl">;
1130 ////////////////////////////////////////////////////////////////////////////////
1131 // Scalar Shift (Immediate)
1132 let isScalarShift = 1 in {
1133 // Signed/Unsigned Shift Right (Immediate)
1134 def SCALAR_SSHR_N: SInst<"vshr_n", "ssi", "SlSUl">;
1135 // Signed/Unsigned Rounding Shift Right (Immediate)
1136 def SCALAR_SRSHR_N: SInst<"vrshr_n", "ssi", "SlSUl">;
1138 // Signed/Unsigned Shift Right and Accumulate (Immediate)
1139 def SCALAR_SSRA_N: SInst<"vsra_n", "sssi", "SlSUl">;
1140 // Signed/Unsigned Rounding Shift Right and Accumulate (Immediate)
1141 def SCALAR_SRSRA_N: SInst<"vrsra_n", "sssi", "SlSUl">;
1143 // Shift Left (Immediate)
1144 def SCALAR_SHL_N: SInst<"vshl_n", "ssi", "SlSUl">;
1145 // Signed/Unsigned Saturating Shift Left (Immediate)
1146 def SCALAR_SQSHL_N: SInst<"vqshl_n", "ssi", "ScSsSiSlSUcSUsSUiSUl">;
1147 // Signed Saturating Shift Left Unsigned (Immediate)
1148 def SCALAR_SQSHLU_N: SInst<"vqshlu_n", "ssi", "ScSsSiSl">;
1150 // Shift Right And Insert (Immediate)
1151 def SCALAR_SRI_N: SInst<"vsri_n", "sssi", "SlSUl">;
1152 // Shift Left And Insert (Immediate)
1153 def SCALAR_SLI_N: SInst<"vsli_n", "sssi", "SlSUl">;
1155 let isScalarNarrowShift = 1 in {
1156 // Signed/Unsigned Saturating Shift Right Narrow (Immediate)
1157 def SCALAR_SQSHRN_N: SInst<"vqshrn_n", "zsi", "SsSiSlSUsSUiSUl">;
1158 // Signed/Unsigned Saturating Rounded Shift Right Narrow (Immediate)
1159 def SCALAR_SQRSHRN_N: SInst<"vqrshrn_n", "zsi", "SsSiSlSUsSUiSUl">;
1160 // Signed Saturating Shift Right Unsigned Narrow (Immediate)
1161 def SCALAR_SQSHRUN_N: SInst<"vqshrun_n", "zsi", "SsSiSl">;
1162 // Signed Saturating Rounded Shift Right Unsigned Narrow (Immediate)
1163 def SCALAR_SQRSHRUN_N: SInst<"vqrshrun_n", "zsi", "SsSiSl">;
1166 ////////////////////////////////////////////////////////////////////////////////
1167 // Scalar Signed/Unsigned Fixed-point Convert To Floating-Point (Immediate)
1168 def SCALAR_SCVTF_N_F32: SInst<"vcvt_n_f32", "ysi", "SiSUi">;
1169 def SCALAR_SCVTF_N_F64: SInst<"vcvt_n_f64", "osi", "SlSUl">;
1171 ////////////////////////////////////////////////////////////////////////////////
1172 // Scalar Floating-point Convert To Signed/Unsigned Fixed-point (Immediate)
1173 def SCALAR_FCVTZS_N_S32 : SInst<"vcvt_n_s32", "$si", "Sf">;
1174 def SCALAR_FCVTZU_N_U32 : SInst<"vcvt_n_u32", "bsi", "Sf">;
1175 def SCALAR_FCVTZS_N_S64 : SInst<"vcvt_n_s64", "$si", "Sd">;
1176 def SCALAR_FCVTZU_N_U64 : SInst<"vcvt_n_u64", "bsi", "Sd">;
1179 ////////////////////////////////////////////////////////////////////////////////
1180 // Scalar Floating-point Round to Integral
1181 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1182 def SCALAR_FRINTN_S32 : SInst<"vrndn", "ss", "Sf">;
1185 ////////////////////////////////////////////////////////////////////////////////
1186 // Scalar Reduce Pairwise Addition (Scalar and Floating Point)
1187 def SCALAR_ADDP : SInst<"vpadd", "sd", "SfSHlSHdSHUl">;
1189 ////////////////////////////////////////////////////////////////////////////////
1190 // Scalar Reduce Floating Point Pairwise Max/Min
1191 def SCALAR_FMAXP : SInst<"vpmax", "sd", "SfSQd">;
1193 def SCALAR_FMINP : SInst<"vpmin", "sd", "SfSQd">;
1195 ////////////////////////////////////////////////////////////////////////////////
1196 // Scalar Reduce Floating Point Pairwise maxNum/minNum
1197 def SCALAR_FMAXNMP : SInst<"vpmaxnm", "sd", "SfSQd">;
1198 def SCALAR_FMINNMP : SInst<"vpminnm", "sd", "SfSQd">;
1200 ////////////////////////////////////////////////////////////////////////////////
1201 // Scalar Integer Saturating Doubling Multiply Half High
1202 def SCALAR_SQDMULH : SInst<"vqdmulh", "sss", "SsSi">;
1204 ////////////////////////////////////////////////////////////////////////////////
1205 // Scalar Integer Saturating Rounding Doubling Multiply Half High
1206 def SCALAR_SQRDMULH : SInst<"vqrdmulh", "sss", "SsSi">;
1208 let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1209 ////////////////////////////////////////////////////////////////////////////////
1210 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
1211 def SCALAR_SQRDMLAH : SOpInst<"vqrdmlah", "ssss", "SsSi", OP_QRDMLAH>;
1213 ////////////////////////////////////////////////////////////////////////////////
1214 // Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
1215 def SCALAR_SQRDMLSH : SOpInst<"vqrdmlsh", "ssss", "SsSi", OP_QRDMLSH>;
1218 ////////////////////////////////////////////////////////////////////////////////
1219 // Scalar Floating-point Multiply Extended
1220 def SCALAR_FMULX : IInst<"vmulx", "sss", "SfSd">;
1222 ////////////////////////////////////////////////////////////////////////////////
1223 // Scalar Floating-point Reciprocal Step
1224 def SCALAR_FRECPS : IInst<"vrecps", "sss", "SfSd">;
1226 ////////////////////////////////////////////////////////////////////////////////
1227 // Scalar Floating-point Reciprocal Square Root Step
1228 def SCALAR_FRSQRTS : IInst<"vrsqrts", "sss", "SfSd">;
1230 ////////////////////////////////////////////////////////////////////////////////
1231 // Scalar Signed Integer Convert To Floating-point
1232 def SCALAR_SCVTFS : SInst<"vcvt_f32", "ys", "Si">;
1233 def SCALAR_SCVTFD : SInst<"vcvt_f64", "os", "Sl">;
1235 ////////////////////////////////////////////////////////////////////////////////
1236 // Scalar Unsigned Integer Convert To Floating-point
1237 def SCALAR_UCVTFS : SInst<"vcvt_f32", "ys", "SUi">;
1238 def SCALAR_UCVTFD : SInst<"vcvt_f64", "os", "SUl">;
1240 ////////////////////////////////////////////////////////////////////////////////
1241 // Scalar Floating-point Converts
1242 def SCALAR_FCVTXN : IInst<"vcvtx_f32", "ys", "Sd">;
1243 def SCALAR_FCVTNSS : SInst<"vcvtn_s32", "$s", "Sf">;
1244 def SCALAR_FCVTNUS : SInst<"vcvtn_u32", "bs", "Sf">;
1245 def SCALAR_FCVTNSD : SInst<"vcvtn_s64", "$s", "Sd">;
1246 def SCALAR_FCVTNUD : SInst<"vcvtn_u64", "bs", "Sd">;
1247 def SCALAR_FCVTMSS : SInst<"vcvtm_s32", "$s", "Sf">;
1248 def SCALAR_FCVTMUS : SInst<"vcvtm_u32", "bs", "Sf">;
1249 def SCALAR_FCVTMSD : SInst<"vcvtm_s64", "$s", "Sd">;
1250 def SCALAR_FCVTMUD : SInst<"vcvtm_u64", "bs", "Sd">;
1251 def SCALAR_FCVTASS : SInst<"vcvta_s32", "$s", "Sf">;
1252 def SCALAR_FCVTAUS : SInst<"vcvta_u32", "bs", "Sf">;
1253 def SCALAR_FCVTASD : SInst<"vcvta_s64", "$s", "Sd">;
1254 def SCALAR_FCVTAUD : SInst<"vcvta_u64", "bs", "Sd">;
1255 def SCALAR_FCVTPSS : SInst<"vcvtp_s32", "$s", "Sf">;
1256 def SCALAR_FCVTPUS : SInst<"vcvtp_u32", "bs", "Sf">;
1257 def SCALAR_FCVTPSD : SInst<"vcvtp_s64", "$s", "Sd">;
1258 def SCALAR_FCVTPUD : SInst<"vcvtp_u64", "bs", "Sd">;
1259 def SCALAR_FCVTZSS : SInst<"vcvt_s32", "$s", "Sf">;
1260 def SCALAR_FCVTZUS : SInst<"vcvt_u32", "bs", "Sf">;
1261 def SCALAR_FCVTZSD : SInst<"vcvt_s64", "$s", "Sd">;
1262 def SCALAR_FCVTZUD : SInst<"vcvt_u64", "bs", "Sd">;
1264 ////////////////////////////////////////////////////////////////////////////////
1265 // Scalar Floating-point Reciprocal Estimate
1266 def SCALAR_FRECPE : IInst<"vrecpe", "ss", "SfSd">;
1268 ////////////////////////////////////////////////////////////////////////////////
1269 // Scalar Floating-point Reciprocal Exponent
1270 def SCALAR_FRECPX : IInst<"vrecpx", "ss", "SfSd">;
1272 ////////////////////////////////////////////////////////////////////////////////
1273 // Scalar Floating-point Reciprocal Square Root Estimate
1274 def SCALAR_FRSQRTE : IInst<"vrsqrte", "ss", "SfSd">;
1276 ////////////////////////////////////////////////////////////////////////////////
1277 // Scalar Integer Comparison
1278 def SCALAR_CMEQ : SInst<"vceq", "sss", "SlSUl">;
1279 def SCALAR_CMEQZ : SInst<"vceqz", "ss", "SlSUl">;
1280 def SCALAR_CMGE : SInst<"vcge", "sss", "Sl">;
1281 def SCALAR_CMGEZ : SInst<"vcgez", "ss", "Sl">;
1282 def SCALAR_CMHS : SInst<"vcge", "sss", "SUl">;
1283 def SCALAR_CMLE : SInst<"vcle", "sss", "SlSUl">;
1284 def SCALAR_CMLEZ : SInst<"vclez", "ss", "Sl">;
1285 def SCALAR_CMLT : SInst<"vclt", "sss", "SlSUl">;
1286 def SCALAR_CMLTZ : SInst<"vcltz", "ss", "Sl">;
1287 def SCALAR_CMGT : SInst<"vcgt", "sss", "Sl">;
1288 def SCALAR_CMGTZ : SInst<"vcgtz", "ss", "Sl">;
1289 def SCALAR_CMHI : SInst<"vcgt", "sss", "SUl">;
1290 def SCALAR_CMTST : SInst<"vtst", "sss", "SlSUl">;
1292 ////////////////////////////////////////////////////////////////////////////////
1293 // Scalar Floating-point Comparison
1294 def SCALAR_FCMEQ : IInst<"vceq", "bss", "SfSd">;
1295 def SCALAR_FCMEQZ : IInst<"vceqz", "bs", "SfSd">;
1296 def SCALAR_FCMGE : IInst<"vcge", "bss", "SfSd">;
1297 def SCALAR_FCMGEZ : IInst<"vcgez", "bs", "SfSd">;
1298 def SCALAR_FCMGT : IInst<"vcgt", "bss", "SfSd">;
1299 def SCALAR_FCMGTZ : IInst<"vcgtz", "bs", "SfSd">;
1300 def SCALAR_FCMLE : IInst<"vcle", "bss", "SfSd">;
1301 def SCALAR_FCMLEZ : IInst<"vclez", "bs", "SfSd">;
1302 def SCALAR_FCMLT : IInst<"vclt", "bss", "SfSd">;
1303 def SCALAR_FCMLTZ : IInst<"vcltz", "bs", "SfSd">;
1305 ////////////////////////////////////////////////////////////////////////////////
1306 // Scalar Floating-point Absolute Compare Mask Greater Than Or Equal
1307 def SCALAR_FACGE : IInst<"vcage", "bss", "SfSd">;
1308 def SCALAR_FACLE : IInst<"vcale", "bss", "SfSd">;
1310 ////////////////////////////////////////////////////////////////////////////////
1311 // Scalar Floating-point Absolute Compare Mask Greater Than
1312 def SCALAR_FACGT : IInst<"vcagt", "bss", "SfSd">;
1313 def SCALAR_FACLT : IInst<"vcalt", "bss", "SfSd">;
1315 ////////////////////////////////////////////////////////////////////////////////
1316 // Scalar Absolute Value
1317 def SCALAR_ABS : SInst<"vabs", "ss", "Sl">;
1319 ////////////////////////////////////////////////////////////////////////////////
1320 // Scalar Absolute Difference
1321 def SCALAR_ABD : IInst<"vabd", "sss", "SfSd">;
1323 ////////////////////////////////////////////////////////////////////////////////
1324 // Scalar Signed Saturating Absolute Value
1325 def SCALAR_SQABS : SInst<"vqabs", "ss", "ScSsSiSl">;
1327 ////////////////////////////////////////////////////////////////////////////////
1329 def SCALAR_NEG : SInst<"vneg", "ss", "Sl">;
1331 ////////////////////////////////////////////////////////////////////////////////
1332 // Scalar Signed Saturating Negate
1333 def SCALAR_SQNEG : SInst<"vqneg", "ss", "ScSsSiSl">;
1335 ////////////////////////////////////////////////////////////////////////////////
1336 // Scalar Signed Saturating Accumulated of Unsigned Value
1337 def SCALAR_SUQADD : SInst<"vuqadd", "sss", "ScSsSiSl">;
1339 ////////////////////////////////////////////////////////////////////////////////
1340 // Scalar Unsigned Saturating Accumulated of Signed Value
1341 def SCALAR_USQADD : SInst<"vsqadd", "sss", "SUcSUsSUiSUl">;
1343 ////////////////////////////////////////////////////////////////////////////////
1344 // Signed Saturating Doubling Multiply-Add Long
1345 def SCALAR_SQDMLAL : SInst<"vqdmlal", "rrss", "SsSi">;
1347 ////////////////////////////////////////////////////////////////////////////////
1348 // Signed Saturating Doubling Multiply-Subtract Long
1349 def SCALAR_SQDMLSL : SInst<"vqdmlsl", "rrss", "SsSi">;
1351 ////////////////////////////////////////////////////////////////////////////////
1352 // Signed Saturating Doubling Multiply Long
1353 def SCALAR_SQDMULL : SInst<"vqdmull", "rss", "SsSi">;
1355 ////////////////////////////////////////////////////////////////////////////////
1356 // Scalar Signed Saturating Extract Unsigned Narrow
1357 def SCALAR_SQXTUN : SInst<"vqmovun", "zs", "SsSiSl">;
1359 ////////////////////////////////////////////////////////////////////////////////
1360 // Scalar Signed Saturating Extract Narrow
1361 def SCALAR_SQXTN : SInst<"vqmovn", "zs", "SsSiSl">;
1363 ////////////////////////////////////////////////////////////////////////////////
1364 // Scalar Unsigned Saturating Extract Narrow
1365 def SCALAR_UQXTN : SInst<"vqmovn", "zs", "SUsSUiSUl">;
1367 // Scalar Floating Point multiply (scalar, by element)
1368 def SCALAR_FMUL_LANE : IOpInst<"vmul_lane", "ssdi", "SfSd", OP_SCALAR_MUL_LN>;
1369 def SCALAR_FMUL_LANEQ : IOpInst<"vmul_laneq", "ssji", "SfSd", OP_SCALAR_MUL_LN>;
1371 // Scalar Floating Point multiply extended (scalar, by element)
1372 def SCALAR_FMULX_LANE : IOpInst<"vmulx_lane", "ssdi", "SfSd", OP_SCALAR_MULX_LN>;
1373 def SCALAR_FMULX_LANEQ : IOpInst<"vmulx_laneq", "ssji", "SfSd", OP_SCALAR_MULX_LN>;
1375 def SCALAR_VMUL_N : IInst<"vmul_n", "dds", "d">;
1377 // VMUL_LANE_A64 d type implemented using scalar mul lane
1378 def SCALAR_VMUL_LANE : IInst<"vmul_lane", "ddgi", "d">;
1380 // VMUL_LANEQ d type implemented using scalar mul lane
1381 def SCALAR_VMUL_LANEQ : IInst<"vmul_laneq", "ddji", "d"> {
1385 // VMULX_LANE d type implemented using scalar vmulx_lane
1386 def SCALAR_VMULX_LANE : IOpInst<"vmulx_lane", "ddgi", "d", OP_SCALAR_VMULX_LN>;
1388 // VMULX_LANEQ d type implemented using scalar vmulx_laneq
1389 def SCALAR_VMULX_LANEQ : IOpInst<"vmulx_laneq", "ddji", "d", OP_SCALAR_VMULX_LNQ>;
1391 // Scalar Floating Point fused multiply-add (scalar, by element)
1392 def SCALAR_FMLA_LANE : IInst<"vfma_lane", "sssdi", "SfSd">;
1393 def SCALAR_FMLA_LANEQ : IInst<"vfma_laneq", "sssji", "SfSd">;
1395 // Scalar Floating Point fused multiply-subtract (scalar, by element)
1396 def SCALAR_FMLS_LANE : IOpInst<"vfms_lane", "sssdi", "SfSd", OP_FMS_LN>;
1397 def SCALAR_FMLS_LANEQ : IOpInst<"vfms_laneq", "sssji", "SfSd", OP_FMS_LNQ>;
1399 // Signed Saturating Doubling Multiply Long (scalar by element)
1400 def SCALAR_SQDMULL_LANE : SOpInst<"vqdmull_lane", "rsdi", "SsSi", OP_SCALAR_QDMULL_LN>;
1401 def SCALAR_SQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "rsji", "SsSi", OP_SCALAR_QDMULL_LN>;
1403 // Signed Saturating Doubling Multiply-Add Long (scalar by element)
1404 def SCALAR_SQDMLAL_LANE : SInst<"vqdmlal_lane", "rrsdi", "SsSi">;
1405 def SCALAR_SQDMLAL_LANEQ : SInst<"vqdmlal_laneq", "rrsji", "SsSi">;
1407 // Signed Saturating Doubling Multiply-Subtract Long (scalar by element)
1408 def SCALAR_SQDMLS_LANE : SInst<"vqdmlsl_lane", "rrsdi", "SsSi">;
1409 def SCALAR_SQDMLS_LANEQ : SInst<"vqdmlsl_laneq", "rrsji", "SsSi">;
1411 // Scalar Integer Saturating Doubling Multiply Half High (scalar by element)
1412 def SCALAR_SQDMULH_LANE : SOpInst<"vqdmulh_lane", "ssdi", "SsSi", OP_SCALAR_QDMULH_LN>;
1413 def SCALAR_SQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "ssji", "SsSi", OP_SCALAR_QDMULH_LN>;
1415 // Scalar Integer Saturating Rounding Doubling Multiply Half High
1416 def SCALAR_SQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "ssdi", "SsSi", OP_SCALAR_QRDMULH_LN>;
1417 def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "ssji", "SsSi", OP_SCALAR_QRDMULH_LN>;
1419 let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1420 // Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
1421 def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "sssdi", "SsSi", OP_SCALAR_QRDMLAH_LN>;
1422 def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "sssji", "SsSi", OP_SCALAR_QRDMLAH_LN>;
1424 // Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
1425 def SCALAR_SQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "sssdi", "SsSi", OP_SCALAR_QRDMLSH_LN>;
1426 def SCALAR_SQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "sssji", "SsSi", OP_SCALAR_QRDMLSH_LN>;
1429 def SCALAR_VDUP_LANE : IInst<"vdup_lane", "sdi", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
1430 def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "sji", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
1433 // ARMv8.2-A FP16 vector intrinsics for A32/A64.
1434 let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1436 // ARMv8.2-A FP16 one-operand vector intrinsics.
1439 def CMEQH : SInst<"vceqz", "ud", "hQh">;
1440 def CMGEH : SInst<"vcgez", "ud", "hQh">;
1441 def CMGTH : SInst<"vcgtz", "ud", "hQh">;
1442 def CMLEH : SInst<"vclez", "ud", "hQh">;
1443 def CMLTH : SInst<"vcltz", "ud", "hQh">;
1445 // Vector conversion
1446 def VCVT_F16 : SInst<"vcvt_f16", "Hd", "sUsQsQUs">;
1447 def VCVT_S16 : SInst<"vcvt_s16", "xd", "hQh">;
1448 def VCVT_U16 : SInst<"vcvt_u16", "ud", "hQh">;
1449 def VCVTA_S16 : SInst<"vcvta_s16", "xd", "hQh">;
1450 def VCVTA_U16 : SInst<"vcvta_u16", "ud", "hQh">;
1451 def VCVTM_S16 : SInst<"vcvtm_s16", "xd", "hQh">;
1452 def VCVTM_U16 : SInst<"vcvtm_u16", "ud", "hQh">;
1453 def VCVTN_S16 : SInst<"vcvtn_s16", "xd", "hQh">;
1454 def VCVTN_U16 : SInst<"vcvtn_u16", "ud", "hQh">;
1455 def VCVTP_S16 : SInst<"vcvtp_s16", "xd", "hQh">;
1456 def VCVTP_U16 : SInst<"vcvtp_u16", "ud", "hQh">;
1459 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1460 def FRINTZH : SInst<"vrnd", "dd", "hQh">;
1461 def FRINTNH : SInst<"vrndn", "dd", "hQh">;
1462 def FRINTAH : SInst<"vrnda", "dd", "hQh">;
1463 def FRINTPH : SInst<"vrndp", "dd", "hQh">;
1464 def FRINTMH : SInst<"vrndm", "dd", "hQh">;
1465 def FRINTXH : SInst<"vrndx", "dd", "hQh">;
1469 def VABSH : SInst<"vabs", "dd", "hQh">;
1470 def VNEGH : SOpInst<"vneg", "dd", "hQh", OP_NEG>;
1471 def VRECPEH : SInst<"vrecpe", "dd", "hQh">;
1472 def FRSQRTEH : SInst<"vrsqrte", "dd", "hQh">;
1474 // ARMv8.2-A FP16 two-operands vector intrinsics.
1477 def VADDH : SOpInst<"vadd", "ddd", "hQh", OP_ADD>;
1478 def VABDH : SInst<"vabd", "ddd", "hQh">;
1479 def VSUBH : SOpInst<"vsub", "ddd", "hQh", OP_SUB>;
1482 let InstName = "vacge" in {
1483 def VCAGEH : SInst<"vcage", "udd", "hQh">;
1484 def VCALEH : SInst<"vcale", "udd", "hQh">;
1486 let InstName = "vacgt" in {
1487 def VCAGTH : SInst<"vcagt", "udd", "hQh">;
1488 def VCALTH : SInst<"vcalt", "udd", "hQh">;
1490 def VCEQH : SOpInst<"vceq", "udd", "hQh", OP_EQ>;
1491 def VCGEH : SOpInst<"vcge", "udd", "hQh", OP_GE>;
1492 def VCGTH : SOpInst<"vcgt", "udd", "hQh", OP_GT>;
1493 let InstName = "vcge" in
1494 def VCLEH : SOpInst<"vcle", "udd", "hQh", OP_LE>;
1495 let InstName = "vcgt" in
1496 def VCLTH : SOpInst<"vclt", "udd", "hQh", OP_LT>;
1498 // Vector conversion
1499 let isVCVT_N = 1 in {
1500 def VCVT_N_F16 : SInst<"vcvt_n_f16", "Hdi", "sUsQsQUs">;
1501 def VCVT_N_S16 : SInst<"vcvt_n_s16", "xdi", "hQh">;
1502 def VCVT_N_U16 : SInst<"vcvt_n_u16", "udi", "hQh">;
1506 def VMAXH : SInst<"vmax", "ddd", "hQh">;
1507 def VMINH : SInst<"vmin", "ddd", "hQh">;
1508 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1509 def FMAXNMH : SInst<"vmaxnm", "ddd", "hQh">;
1510 def FMINNMH : SInst<"vminnm", "ddd", "hQh">;
1513 // Multiplication/Division
1514 def VMULH : SOpInst<"vmul", "ddd", "hQh", OP_MUL>;
1516 // Pairwise addition
1517 def VPADDH : SInst<"vpadd", "ddd", "h">;
1520 def VPMAXH : SInst<"vpmax", "ddd", "h">;
1521 def VPMINH : SInst<"vpmin", "ddd", "h">;
1524 def VRECPSH : SInst<"vrecps", "ddd", "hQh">;
1525 def VRSQRTSH : SInst<"vrsqrts", "ddd", "hQh">;
1527 // ARMv8.2-A FP16 three-operands vector intrinsics.
1529 // Vector fused multiply-add operations
1530 def VFMAH : SInst<"vfma", "dddd", "hQh">;
1531 def VFMSH : SOpInst<"vfms", "dddd", "hQh", OP_FMLS>;
1533 // ARMv8.2-A FP16 lane vector intrinsics.
1536 def VMUL_LANEH : IOpInst<"vmul_lane", "ddgi", "hQh", OP_MUL_LN>;
1537 def VMUL_NH : IOpInst<"vmul_n", "dds", "hQh", OP_MUL_N>;
1539 // Data processing intrinsics - section 5
1541 // Logical operations
1542 let isHiddenLInst = 1 in
1543 def VBSLH : SInst<"vbsl", "dudd", "hQh">;
1545 // Transposition operations
1546 def VZIPH : WInst<"vzip", "2dd", "hQh">;
1547 def VUZPH : WInst<"vuzp", "2dd", "hQh">;
1548 def VTRNH : WInst<"vtrn", "2dd", "hQh">;
1551 let ArchGuard = "!defined(__aarch64__)" in {
1552 // Set all lanes to same value.
1553 // Already implemented prior to ARMv8.2-A.
1554 def VMOV_NH : WOpInst<"vmov_n", "ds", "hQh", OP_DUP>;
1555 def VDUP_NH : WOpInst<"vdup_n", "ds", "hQh", OP_DUP>;
1556 def VDUP_LANE1H : WOpInst<"vdup_lane", "dgi", "hQh", OP_DUP_LN>;
1560 def VEXTH : WInst<"vext", "dddi", "hQh">;
1562 // Reverse vector elements
1563 def VREV64H : WOpInst<"vrev64", "dd", "hQh", OP_REV64>;
1566 // ARMv8.2-A FP16 vector intrinsics for A64 only.
1567 let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in {
1570 def FRINTIH : SInst<"vrndi", "dd", "hQh">;
1573 def FSQRTH : SInst<"vsqrt", "dd", "hQh">;
1575 // Multiplication/Division
1576 def MULXH : SInst<"vmulx", "ddd", "hQh">;
1577 def FDIVH : IOpInst<"vdiv", "ddd", "hQh", OP_DIV>;
1579 // Pairwise addition
1580 def VPADDH1 : SInst<"vpadd", "ddd", "Qh">;
1583 def VPMAXH1 : SInst<"vpmax", "ddd", "Qh">;
1584 def VPMINH1 : SInst<"vpmin", "ddd", "Qh">;
1586 // Pairwise MaxNum/MinNum
1587 def FMAXNMPH : SInst<"vpmaxnm", "ddd", "hQh">;
1588 def FMINNMPH : SInst<"vpminnm", "ddd", "hQh">;
1590 // ARMv8.2-A FP16 lane vector intrinsics.
1593 def VFMA_LANEH : IInst<"vfma_lane", "dddgi", "hQh">;
1594 def VFMA_LANEQH : IInst<"vfma_laneq", "dddji", "hQh">;
1596 // FMA lane with scalar argument
1597 def FMLA_NH : SOpInst<"vfma_n", "ddds", "hQh", OP_FMLA_N>;
1598 // Scalar floating point fused multiply-add (scalar, by element)
1599 def SCALAR_FMLA_LANEH : IInst<"vfma_lane", "sssdi", "Sh">;
1600 def SCALAR_FMLA_LANEQH : IInst<"vfma_laneq", "sssji", "Sh">;
1603 def VFMS_LANEH : IOpInst<"vfms_lane", "dddgi", "hQh", OP_FMS_LN>;
1604 def VFMS_LANEQH : IOpInst<"vfms_laneq", "dddji", "hQh", OP_FMS_LNQ>;
1605 // FMS lane with scalar argument
1606 def FMLS_NH : SOpInst<"vfms_n", "ddds", "hQh", OP_FMLS_N>;
1607 // Scalar floating foint fused multiply-subtract (scalar, by element)
1608 def SCALAR_FMLS_LANEH : IOpInst<"vfms_lane", "sssdi", "Sh", OP_FMS_LN>;
1609 def SCALAR_FMLS_LANEQH : IOpInst<"vfms_laneq", "sssji", "Sh", OP_FMS_LNQ>;
1612 def VMUL_LANEQH : IOpInst<"vmul_laneq", "ddji", "hQh", OP_MUL_LN>;
1613 // Scalar floating point multiply (scalar, by element)
1614 def SCALAR_FMUL_LANEH : IOpInst<"vmul_lane", "ssdi", "Sh", OP_SCALAR_MUL_LN>;
1615 def SCALAR_FMUL_LANEQH : IOpInst<"vmul_laneq", "ssji", "Sh", OP_SCALAR_MUL_LN>;
1618 def VMULX_LANEH : IOpInst<"vmulx_lane", "ddgi", "hQh", OP_MULX_LN>;
1619 def VMULX_LANEQH : IOpInst<"vmulx_laneq", "ddji", "hQh", OP_MULX_LN>;
1620 def VMULX_NH : IOpInst<"vmulx_n", "dds", "hQh", OP_MULX_N>;
1621 // Scalar floating point mulx (scalar, by element)
1622 def SCALAR_FMULX_LANEH : IInst<"vmulx_lane", "ssdi", "Sh">;
1623 def SCALAR_FMULX_LANEQH : IInst<"vmulx_laneq", "ssji", "Sh">;
1625 // ARMv8.2-A FP16 reduction vector intrinsics.
1626 def VMAXVH : SInst<"vmaxv", "sd", "hQh">;
1627 def VMINVH : SInst<"vminv", "sd", "hQh">;
1628 def FMAXNMVH : SInst<"vmaxnmv", "sd", "hQh">;
1629 def FMINNMVH : SInst<"vminnmv", "sd", "hQh">;
1632 def VTRN1H : SOpInst<"vtrn1", "ddd", "hQh", OP_TRN1>;
1633 def VZIP1H : SOpInst<"vzip1", "ddd", "hQh", OP_ZIP1>;
1634 def VUZP1H : SOpInst<"vuzp1", "ddd", "hQh", OP_UZP1>;
1635 def VTRN2H : SOpInst<"vtrn2", "ddd", "hQh", OP_TRN2>;
1636 def VZIP2H : SOpInst<"vzip2", "ddd", "hQh", OP_ZIP2>;
1637 def VUZP2H : SOpInst<"vuzp2", "ddd", "hQh", OP_UZP2>;
1639 def SCALAR_VDUP_LANEH : IInst<"vdup_lane", "sdi", "Sh">;
1640 def SCALAR_VDUP_LANEQH : IInst<"vdup_laneq", "sji", "Sh">;
1643 // v8.2-A dot product instructions.
1644 let ArchGuard = "defined(__ARM_FEATURE_DOTPROD)" in {
1645 def DOT : SInst<"vdot", "dd88", "iQiUiQUi">;
1646 def DOT_LANE : SOpInst<"vdot_lane", "dd87i", "iUiQiQUi", OP_DOT_LN>;
1648 let ArchGuard = "defined(__ARM_FEATURE_DOTPROD) && defined(__aarch64__)" in {
1649 // Variants indexing into a 128-bit vector are A64 only.
1650 def UDOT_LANEQ : SOpInst<"vdot_laneq", "dd89i", "iUiQiQUi", OP_DOT_LNQ>;
1653 // v8.2-A FP16 fused multiply-add long instructions.
1654 let ArchGuard = "defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__)" in {
1655 def VFMLAL_LOW : SInst<"vfmlal_low", "ffHH", "UiQUi">;
1656 def VFMLSL_LOW : SInst<"vfmlsl_low", "ffHH", "UiQUi">;
1657 def VFMLAL_HIGH : SInst<"vfmlal_high", "ffHH", "UiQUi">;
1658 def VFMLSL_HIGH : SInst<"vfmlsl_high", "ffHH", "UiQUi">;
1660 def VFMLAL_LANE_LOW : SOpInst<"vfmlal_lane_low", "ffH0i", "UiQUi", OP_FMLAL_LN>;
1661 def VFMLSL_LANE_LOW : SOpInst<"vfmlsl_lane_low", "ffH0i", "UiQUi", OP_FMLSL_LN>;
1662 def VFMLAL_LANE_HIGH : SOpInst<"vfmlal_lane_high", "ffH0i", "UiQUi", OP_FMLAL_LN_Hi>;
1663 def VFMLSL_LANE_HIGH : SOpInst<"vfmlsl_lane_high", "ffH0i", "UiQUi", OP_FMLSL_LN_Hi>;
1665 def VFMLAL_LANEQ_LOW : SOpInst<"vfmlal_laneq_low", "ffH1i", "UiQUi", OP_FMLAL_LN>;
1666 def VFMLSL_LANEQ_LOW : SOpInst<"vfmlsl_laneq_low", "ffH1i", "UiQUi", OP_FMLSL_LN>;
1667 def VFMLAL_LANEQ_HIGH : SOpInst<"vfmlal_laneq_high", "ffH1i", "UiQUi", OP_FMLAL_LN_Hi>;
1668 def VFMLSL_LANEQ_HIGH : SOpInst<"vfmlsl_laneq_high", "ffH1i", "UiQUi", OP_FMLSL_LN_Hi>;