1 //=- AArch64InstrInfo.td - Describe the AArch64 Instructions -*- tablegen -*-=//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // AArch64 Instruction definitions.
12 //===----------------------------------------------------------------------===//
14 //===----------------------------------------------------------------------===//
15 // ARM Instruction Predicate Definitions.
17 def HasV8_1a : Predicate<"Subtarget->hasV8_1aOps()">,
18 AssemblerPredicate<"HasV8_1aOps", "armv8.1a">;
19 def HasV8_2a : Predicate<"Subtarget->hasV8_2aOps()">,
20 AssemblerPredicate<"HasV8_2aOps", "armv8.2a">;
21 def HasV8_3a : Predicate<"Subtarget->hasV8_3aOps()">,
22 AssemblerPredicate<"HasV8_3aOps", "armv8.3a">;
23 def HasFPARMv8 : Predicate<"Subtarget->hasFPARMv8()">,
24 AssemblerPredicate<"FeatureFPARMv8", "fp-armv8">;
25 def HasNEON : Predicate<"Subtarget->hasNEON()">,
26 AssemblerPredicate<"FeatureNEON", "neon">;
27 def HasCrypto : Predicate<"Subtarget->hasCrypto()">,
28 AssemblerPredicate<"FeatureCrypto", "crypto">;
29 def HasDotProd : Predicate<"Subtarget->hasDotProd()">,
30 AssemblerPredicate<"FeatureDotProd", "dotprod">;
31 def HasCRC : Predicate<"Subtarget->hasCRC()">,
32 AssemblerPredicate<"FeatureCRC", "crc">;
33 def HasLSE : Predicate<"Subtarget->hasLSE()">,
34 AssemblerPredicate<"FeatureLSE", "lse">;
35 def HasRAS : Predicate<"Subtarget->hasRAS()">,
36 AssemblerPredicate<"FeatureRAS", "ras">;
37 def HasRDM : Predicate<"Subtarget->hasRDM()">,
38 AssemblerPredicate<"FeatureRDM", "rdm">;
39 def HasPerfMon : Predicate<"Subtarget->hasPerfMon()">;
40 def HasFullFP16 : Predicate<"Subtarget->hasFullFP16()">,
41 AssemblerPredicate<"FeatureFullFP16", "fullfp16">;
42 def HasSPE : Predicate<"Subtarget->hasSPE()">,
43 AssemblerPredicate<"FeatureSPE", "spe">;
44 def HasFuseAES : Predicate<"Subtarget->hasFuseAES()">,
45 AssemblerPredicate<"FeatureFuseAES",
47 def HasSVE : Predicate<"Subtarget->hasSVE()">,
48 AssemblerPredicate<"FeatureSVE", "sve">;
49 def HasRCPC : Predicate<"Subtarget->hasRCPC()">,
50 AssemblerPredicate<"FeatureRCPC", "rcpc">;
52 def IsLE : Predicate<"Subtarget->isLittleEndian()">;
53 def IsBE : Predicate<"!Subtarget->isLittleEndian()">;
54 def UseAlternateSExtLoadCVTF32
55 : Predicate<"Subtarget->useAlternateSExtLoadCVTF32Pattern()">;
57 def UseNegativeImmediates
58 : Predicate<"false">, AssemblerPredicate<"!FeatureNoNegativeImmediates",
59 "NegativeImmediates">;
62 //===----------------------------------------------------------------------===//
63 // AArch64-specific DAG Nodes.
66 // SDTBinaryArithWithFlagsOut - RES1, FLAGS = op LHS, RHS
67 def SDTBinaryArithWithFlagsOut : SDTypeProfile<2, 2,
70 SDTCisInt<0>, SDTCisVT<1, i32>]>;
72 // SDTBinaryArithWithFlagsIn - RES1, FLAGS = op LHS, RHS, FLAGS
73 def SDTBinaryArithWithFlagsIn : SDTypeProfile<1, 3,
79 // SDTBinaryArithWithFlagsInOut - RES1, FLAGS = op LHS, RHS, FLAGS
80 def SDTBinaryArithWithFlagsInOut : SDTypeProfile<2, 3,
87 def SDT_AArch64Brcond : SDTypeProfile<0, 3,
88 [SDTCisVT<0, OtherVT>, SDTCisVT<1, i32>,
90 def SDT_AArch64cbz : SDTypeProfile<0, 2, [SDTCisInt<0>, SDTCisVT<1, OtherVT>]>;
91 def SDT_AArch64tbz : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>,
92 SDTCisVT<2, OtherVT>]>;
95 def SDT_AArch64CSel : SDTypeProfile<1, 4,
100 def SDT_AArch64CCMP : SDTypeProfile<1, 5,
107 def SDT_AArch64FCCMP : SDTypeProfile<1, 5,
114 def SDT_AArch64FCmp : SDTypeProfile<0, 2,
116 SDTCisSameAs<0, 1>]>;
117 def SDT_AArch64Dup : SDTypeProfile<1, 1, [SDTCisVec<0>]>;
118 def SDT_AArch64DupLane : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<2>]>;
119 def SDT_AArch64Zip : SDTypeProfile<1, 2, [SDTCisVec<0>,
121 SDTCisSameAs<0, 2>]>;
122 def SDT_AArch64MOVIedit : SDTypeProfile<1, 1, [SDTCisInt<1>]>;
123 def SDT_AArch64MOVIshift : SDTypeProfile<1, 2, [SDTCisInt<1>, SDTCisInt<2>]>;
124 def SDT_AArch64vecimm : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
125 SDTCisInt<2>, SDTCisInt<3>]>;
126 def SDT_AArch64UnaryVec: SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
127 def SDT_AArch64ExtVec: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
128 SDTCisSameAs<0,2>, SDTCisInt<3>]>;
129 def SDT_AArch64vshift : SDTypeProfile<1, 2, [SDTCisSameAs<0,1>, SDTCisInt<2>]>;
131 def SDT_AArch64unvec : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
132 def SDT_AArch64fcmpz : SDTypeProfile<1, 1, []>;
133 def SDT_AArch64fcmp : SDTypeProfile<1, 2, [SDTCisSameAs<1,2>]>;
134 def SDT_AArch64binvec : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
136 def SDT_AArch64trivec : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
139 def SDT_AArch64TCRET : SDTypeProfile<0, 2, [SDTCisPtrTy<0>]>;
140 def SDT_AArch64PREFETCH : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>]>;
142 def SDT_AArch64ITOF : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisSameAs<0,1>]>;
144 def SDT_AArch64TLSDescCall : SDTypeProfile<0, -2, [SDTCisPtrTy<0>,
147 // Generates the general dynamic sequences, i.e.
148 // adrp x0, :tlsdesc:var
149 // ldr x1, [x0, #:tlsdesc_lo12:var]
150 // add x0, x0, #:tlsdesc_lo12:var
154 // (the TPIDR_EL0 offset is put directly in X0, hence no "result" here)
155 // number of operands (the variable)
156 def SDT_AArch64TLSDescCallSeq : SDTypeProfile<0,1,
159 def SDT_AArch64WrapperLarge : SDTypeProfile<1, 4,
160 [SDTCisVT<0, i64>, SDTCisVT<1, i32>,
161 SDTCisSameAs<1, 2>, SDTCisSameAs<1, 3>,
162 SDTCisSameAs<1, 4>]>;
166 def AArch64adrp : SDNode<"AArch64ISD::ADRP", SDTIntUnaryOp, []>;
167 def AArch64addlow : SDNode<"AArch64ISD::ADDlow", SDTIntBinOp, []>;
168 def AArch64LOADgot : SDNode<"AArch64ISD::LOADgot", SDTIntUnaryOp>;
169 def AArch64callseq_start : SDNode<"ISD::CALLSEQ_START",
170 SDCallSeqStart<[ SDTCisVT<0, i32>,
172 [SDNPHasChain, SDNPOutGlue]>;
173 def AArch64callseq_end : SDNode<"ISD::CALLSEQ_END",
174 SDCallSeqEnd<[ SDTCisVT<0, i32>,
176 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
177 def AArch64call : SDNode<"AArch64ISD::CALL",
178 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
179 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
181 def AArch64brcond : SDNode<"AArch64ISD::BRCOND", SDT_AArch64Brcond,
183 def AArch64cbz : SDNode<"AArch64ISD::CBZ", SDT_AArch64cbz,
185 def AArch64cbnz : SDNode<"AArch64ISD::CBNZ", SDT_AArch64cbz,
187 def AArch64tbz : SDNode<"AArch64ISD::TBZ", SDT_AArch64tbz,
189 def AArch64tbnz : SDNode<"AArch64ISD::TBNZ", SDT_AArch64tbz,
193 def AArch64csel : SDNode<"AArch64ISD::CSEL", SDT_AArch64CSel>;
194 def AArch64csinv : SDNode<"AArch64ISD::CSINV", SDT_AArch64CSel>;
195 def AArch64csneg : SDNode<"AArch64ISD::CSNEG", SDT_AArch64CSel>;
196 def AArch64csinc : SDNode<"AArch64ISD::CSINC", SDT_AArch64CSel>;
197 def AArch64retflag : SDNode<"AArch64ISD::RET_FLAG", SDTNone,
198 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
199 def AArch64adc : SDNode<"AArch64ISD::ADC", SDTBinaryArithWithFlagsIn >;
200 def AArch64sbc : SDNode<"AArch64ISD::SBC", SDTBinaryArithWithFlagsIn>;
201 def AArch64add_flag : SDNode<"AArch64ISD::ADDS", SDTBinaryArithWithFlagsOut,
203 def AArch64sub_flag : SDNode<"AArch64ISD::SUBS", SDTBinaryArithWithFlagsOut>;
204 def AArch64and_flag : SDNode<"AArch64ISD::ANDS", SDTBinaryArithWithFlagsOut,
206 def AArch64adc_flag : SDNode<"AArch64ISD::ADCS", SDTBinaryArithWithFlagsInOut>;
207 def AArch64sbc_flag : SDNode<"AArch64ISD::SBCS", SDTBinaryArithWithFlagsInOut>;
209 def AArch64ccmp : SDNode<"AArch64ISD::CCMP", SDT_AArch64CCMP>;
210 def AArch64ccmn : SDNode<"AArch64ISD::CCMN", SDT_AArch64CCMP>;
211 def AArch64fccmp : SDNode<"AArch64ISD::FCCMP", SDT_AArch64FCCMP>;
213 def AArch64threadpointer : SDNode<"AArch64ISD::THREAD_POINTER", SDTPtrLeaf>;
215 def AArch64fcmp : SDNode<"AArch64ISD::FCMP", SDT_AArch64FCmp>;
217 def AArch64dup : SDNode<"AArch64ISD::DUP", SDT_AArch64Dup>;
218 def AArch64duplane8 : SDNode<"AArch64ISD::DUPLANE8", SDT_AArch64DupLane>;
219 def AArch64duplane16 : SDNode<"AArch64ISD::DUPLANE16", SDT_AArch64DupLane>;
220 def AArch64duplane32 : SDNode<"AArch64ISD::DUPLANE32", SDT_AArch64DupLane>;
221 def AArch64duplane64 : SDNode<"AArch64ISD::DUPLANE64", SDT_AArch64DupLane>;
223 def AArch64zip1 : SDNode<"AArch64ISD::ZIP1", SDT_AArch64Zip>;
224 def AArch64zip2 : SDNode<"AArch64ISD::ZIP2", SDT_AArch64Zip>;
225 def AArch64uzp1 : SDNode<"AArch64ISD::UZP1", SDT_AArch64Zip>;
226 def AArch64uzp2 : SDNode<"AArch64ISD::UZP2", SDT_AArch64Zip>;
227 def AArch64trn1 : SDNode<"AArch64ISD::TRN1", SDT_AArch64Zip>;
228 def AArch64trn2 : SDNode<"AArch64ISD::TRN2", SDT_AArch64Zip>;
230 def AArch64movi_edit : SDNode<"AArch64ISD::MOVIedit", SDT_AArch64MOVIedit>;
231 def AArch64movi_shift : SDNode<"AArch64ISD::MOVIshift", SDT_AArch64MOVIshift>;
232 def AArch64movi_msl : SDNode<"AArch64ISD::MOVImsl", SDT_AArch64MOVIshift>;
233 def AArch64mvni_shift : SDNode<"AArch64ISD::MVNIshift", SDT_AArch64MOVIshift>;
234 def AArch64mvni_msl : SDNode<"AArch64ISD::MVNImsl", SDT_AArch64MOVIshift>;
235 def AArch64movi : SDNode<"AArch64ISD::MOVI", SDT_AArch64MOVIedit>;
236 def AArch64fmov : SDNode<"AArch64ISD::FMOV", SDT_AArch64MOVIedit>;
238 def AArch64rev16 : SDNode<"AArch64ISD::REV16", SDT_AArch64UnaryVec>;
239 def AArch64rev32 : SDNode<"AArch64ISD::REV32", SDT_AArch64UnaryVec>;
240 def AArch64rev64 : SDNode<"AArch64ISD::REV64", SDT_AArch64UnaryVec>;
241 def AArch64ext : SDNode<"AArch64ISD::EXT", SDT_AArch64ExtVec>;
243 def AArch64vashr : SDNode<"AArch64ISD::VASHR", SDT_AArch64vshift>;
244 def AArch64vlshr : SDNode<"AArch64ISD::VLSHR", SDT_AArch64vshift>;
245 def AArch64vshl : SDNode<"AArch64ISD::VSHL", SDT_AArch64vshift>;
246 def AArch64sqshli : SDNode<"AArch64ISD::SQSHL_I", SDT_AArch64vshift>;
247 def AArch64uqshli : SDNode<"AArch64ISD::UQSHL_I", SDT_AArch64vshift>;
248 def AArch64sqshlui : SDNode<"AArch64ISD::SQSHLU_I", SDT_AArch64vshift>;
249 def AArch64srshri : SDNode<"AArch64ISD::SRSHR_I", SDT_AArch64vshift>;
250 def AArch64urshri : SDNode<"AArch64ISD::URSHR_I", SDT_AArch64vshift>;
252 def AArch64not: SDNode<"AArch64ISD::NOT", SDT_AArch64unvec>;
253 def AArch64bit: SDNode<"AArch64ISD::BIT", SDT_AArch64trivec>;
254 def AArch64bsl: SDNode<"AArch64ISD::BSL", SDT_AArch64trivec>;
256 def AArch64cmeq: SDNode<"AArch64ISD::CMEQ", SDT_AArch64binvec>;
257 def AArch64cmge: SDNode<"AArch64ISD::CMGE", SDT_AArch64binvec>;
258 def AArch64cmgt: SDNode<"AArch64ISD::CMGT", SDT_AArch64binvec>;
259 def AArch64cmhi: SDNode<"AArch64ISD::CMHI", SDT_AArch64binvec>;
260 def AArch64cmhs: SDNode<"AArch64ISD::CMHS", SDT_AArch64binvec>;
262 def AArch64fcmeq: SDNode<"AArch64ISD::FCMEQ", SDT_AArch64fcmp>;
263 def AArch64fcmge: SDNode<"AArch64ISD::FCMGE", SDT_AArch64fcmp>;
264 def AArch64fcmgt: SDNode<"AArch64ISD::FCMGT", SDT_AArch64fcmp>;
266 def AArch64cmeqz: SDNode<"AArch64ISD::CMEQz", SDT_AArch64unvec>;
267 def AArch64cmgez: SDNode<"AArch64ISD::CMGEz", SDT_AArch64unvec>;
268 def AArch64cmgtz: SDNode<"AArch64ISD::CMGTz", SDT_AArch64unvec>;
269 def AArch64cmlez: SDNode<"AArch64ISD::CMLEz", SDT_AArch64unvec>;
270 def AArch64cmltz: SDNode<"AArch64ISD::CMLTz", SDT_AArch64unvec>;
271 def AArch64cmtst : PatFrag<(ops node:$LHS, node:$RHS),
272 (AArch64not (AArch64cmeqz (and node:$LHS, node:$RHS)))>;
274 def AArch64fcmeqz: SDNode<"AArch64ISD::FCMEQz", SDT_AArch64fcmpz>;
275 def AArch64fcmgez: SDNode<"AArch64ISD::FCMGEz", SDT_AArch64fcmpz>;
276 def AArch64fcmgtz: SDNode<"AArch64ISD::FCMGTz", SDT_AArch64fcmpz>;
277 def AArch64fcmlez: SDNode<"AArch64ISD::FCMLEz", SDT_AArch64fcmpz>;
278 def AArch64fcmltz: SDNode<"AArch64ISD::FCMLTz", SDT_AArch64fcmpz>;
280 def AArch64bici: SDNode<"AArch64ISD::BICi", SDT_AArch64vecimm>;
281 def AArch64orri: SDNode<"AArch64ISD::ORRi", SDT_AArch64vecimm>;
283 def AArch64neg : SDNode<"AArch64ISD::NEG", SDT_AArch64unvec>;
285 def AArch64tcret: SDNode<"AArch64ISD::TC_RETURN", SDT_AArch64TCRET,
286 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
288 def AArch64Prefetch : SDNode<"AArch64ISD::PREFETCH", SDT_AArch64PREFETCH,
289 [SDNPHasChain, SDNPSideEffect]>;
291 def AArch64sitof: SDNode<"AArch64ISD::SITOF", SDT_AArch64ITOF>;
292 def AArch64uitof: SDNode<"AArch64ISD::UITOF", SDT_AArch64ITOF>;
294 def AArch64tlsdesc_callseq : SDNode<"AArch64ISD::TLSDESC_CALLSEQ",
295 SDT_AArch64TLSDescCallSeq,
296 [SDNPInGlue, SDNPOutGlue, SDNPHasChain,
300 def AArch64WrapperLarge : SDNode<"AArch64ISD::WrapperLarge",
301 SDT_AArch64WrapperLarge>;
303 def AArch64NvCast : SDNode<"AArch64ISD::NVCAST", SDTUnaryOp>;
305 def SDT_AArch64mull : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisInt<1>,
306 SDTCisSameAs<1, 2>]>;
307 def AArch64smull : SDNode<"AArch64ISD::SMULL", SDT_AArch64mull>;
308 def AArch64umull : SDNode<"AArch64ISD::UMULL", SDT_AArch64mull>;
310 def AArch64frecpe : SDNode<"AArch64ISD::FRECPE", SDTFPUnaryOp>;
311 def AArch64frecps : SDNode<"AArch64ISD::FRECPS", SDTFPBinOp>;
312 def AArch64frsqrte : SDNode<"AArch64ISD::FRSQRTE", SDTFPUnaryOp>;
313 def AArch64frsqrts : SDNode<"AArch64ISD::FRSQRTS", SDTFPBinOp>;
315 def AArch64saddv : SDNode<"AArch64ISD::SADDV", SDT_AArch64UnaryVec>;
316 def AArch64uaddv : SDNode<"AArch64ISD::UADDV", SDT_AArch64UnaryVec>;
317 def AArch64sminv : SDNode<"AArch64ISD::SMINV", SDT_AArch64UnaryVec>;
318 def AArch64uminv : SDNode<"AArch64ISD::UMINV", SDT_AArch64UnaryVec>;
319 def AArch64smaxv : SDNode<"AArch64ISD::SMAXV", SDT_AArch64UnaryVec>;
320 def AArch64umaxv : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>;
322 //===----------------------------------------------------------------------===//
324 //===----------------------------------------------------------------------===//
326 // AArch64 Instruction Predicate Definitions.
327 // We could compute these on a per-module basis but doing so requires accessing
328 // the Function object through the <Target>Subtarget and objections were raised
329 // to that (see post-commit review comments for r301750).
330 let RecomputePerFunction = 1 in {
331 def ForCodeSize : Predicate<"MF->getFunction().optForSize()">;
332 def NotForCodeSize : Predicate<"!MF->getFunction().optForSize()">;
333 // Avoid generating STRQro if it is slow, unless we're optimizing for code size.
334 def UseSTRQro : Predicate<"!Subtarget->isSTRQroSlow() || MF->getFunction().optForSize()">;
337 include "AArch64InstrFormats.td"
338 include "SVEInstrFormats.td"
340 //===----------------------------------------------------------------------===//
342 //===----------------------------------------------------------------------===//
343 // Miscellaneous instructions.
344 //===----------------------------------------------------------------------===//
346 let Defs = [SP], Uses = [SP], hasSideEffects = 1, isCodeGenOnly = 1 in {
347 // We set Sched to empty list because we expect these instructions to simply get
348 // removed in most cases.
349 def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
350 [(AArch64callseq_start timm:$amt1, timm:$amt2)]>,
352 def ADJCALLSTACKUP : Pseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
353 [(AArch64callseq_end timm:$amt1, timm:$amt2)]>,
355 } // Defs = [SP], Uses = [SP], hasSideEffects = 1, isCodeGenOnly = 1
357 let isReMaterializable = 1, isCodeGenOnly = 1 in {
358 // FIXME: The following pseudo instructions are only needed because remat
359 // cannot handle multiple instructions. When that changes, they can be
360 // removed, along with the AArch64Wrapper node.
362 let AddedComplexity = 10 in
363 def LOADgot : Pseudo<(outs GPR64:$dst), (ins i64imm:$addr),
364 [(set GPR64:$dst, (AArch64LOADgot tglobaladdr:$addr))]>,
367 // The MOVaddr instruction should match only when the add is not folded
368 // into a load or store address.
370 : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
371 [(set GPR64:$dst, (AArch64addlow (AArch64adrp tglobaladdr:$hi),
372 tglobaladdr:$low))]>,
373 Sched<[WriteAdrAdr]>;
375 : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
376 [(set GPR64:$dst, (AArch64addlow (AArch64adrp tjumptable:$hi),
378 Sched<[WriteAdrAdr]>;
380 : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
381 [(set GPR64:$dst, (AArch64addlow (AArch64adrp tconstpool:$hi),
383 Sched<[WriteAdrAdr]>;
385 : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
386 [(set GPR64:$dst, (AArch64addlow (AArch64adrp tblockaddress:$hi),
387 tblockaddress:$low))]>,
388 Sched<[WriteAdrAdr]>;
390 : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
391 [(set GPR64:$dst, (AArch64addlow (AArch64adrp tglobaltlsaddr:$hi),
392 tglobaltlsaddr:$low))]>,
393 Sched<[WriteAdrAdr]>;
395 : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
396 [(set GPR64:$dst, (AArch64addlow (AArch64adrp texternalsym:$hi),
397 texternalsym:$low))]>,
398 Sched<[WriteAdrAdr]>;
400 } // isReMaterializable, isCodeGenOnly
402 def : Pat<(AArch64LOADgot tglobaltlsaddr:$addr),
403 (LOADgot tglobaltlsaddr:$addr)>;
405 def : Pat<(AArch64LOADgot texternalsym:$addr),
406 (LOADgot texternalsym:$addr)>;
408 def : Pat<(AArch64LOADgot tconstpool:$addr),
409 (LOADgot tconstpool:$addr)>;
411 //===----------------------------------------------------------------------===//
412 // System instructions.
413 //===----------------------------------------------------------------------===//
415 def HINT : HintI<"hint">;
416 def : InstAlias<"nop", (HINT 0b000)>;
417 def : InstAlias<"yield",(HINT 0b001)>;
418 def : InstAlias<"wfe", (HINT 0b010)>;
419 def : InstAlias<"wfi", (HINT 0b011)>;
420 def : InstAlias<"sev", (HINT 0b100)>;
421 def : InstAlias<"sevl", (HINT 0b101)>;
422 def : InstAlias<"esb", (HINT 0b10000)>, Requires<[HasRAS]>;
424 // v8.2a Statistical Profiling extension
425 def : InstAlias<"psb $op", (HINT psbhint_op:$op)>, Requires<[HasSPE]>;
427 // As far as LLVM is concerned this writes to the system's exclusive monitors.
428 let mayLoad = 1, mayStore = 1 in
429 def CLREX : CRmSystemI<imm0_15, 0b010, "clrex">;
431 // NOTE: ideally, this would have mayStore = 0, mayLoad = 0, but we cannot
432 // model patterns with sufficiently fine granularity.
433 let mayLoad = ?, mayStore = ? in {
434 def DMB : CRmSystemI<barrier_op, 0b101, "dmb",
435 [(int_aarch64_dmb (i32 imm32_0_15:$CRm))]>;
437 def DSB : CRmSystemI<barrier_op, 0b100, "dsb",
438 [(int_aarch64_dsb (i32 imm32_0_15:$CRm))]>;
440 def ISB : CRmSystemI<barrier_op, 0b110, "isb",
441 [(int_aarch64_isb (i32 imm32_0_15:$CRm))]>;
444 // ARMv8.2 Dot Product
445 let Predicates = [HasDotProd] in {
446 def UDOT2S : BaseSIMDThreeSameVectorDot<0, 1, "udot", ".2s", ".8b">;
447 def SDOT2S : BaseSIMDThreeSameVectorDot<0, 0, "sdot", ".2s", ".8b">;
448 def UDOT4S : BaseSIMDThreeSameVectorDot<1, 1, "udot", ".4s", ".16b">;
449 def SDOT4S : BaseSIMDThreeSameVectorDot<1, 0, "sdot", ".4s", ".16b">;
450 def UDOTIDX2S : BaseSIMDThreeSameVectorDotIndex<0, 1, "udot", ".2s", ".8b", ".4b">;
451 def SDOTIDX2S : BaseSIMDThreeSameVectorDotIndex<0, 0, "sdot", ".2s", ".8b", ".4b">;
452 def UDOTIDX4S : BaseSIMDThreeSameVectorDotIndex<1, 1, "udot", ".4s", ".16b", ".4b">;
453 def SDOTIDX4S : BaseSIMDThreeSameVectorDotIndex<1, 0, "sdot", ".4s", ".16b", ".4b">;
456 let Predicates = [HasRCPC] in {
457 // v8.3 Release Consistent Processor Consistent support, optional in v8.2.
458 def LDAPRB : RCPCLoad<0b00, "ldaprb", GPR32>;
459 def LDAPRH : RCPCLoad<0b01, "ldaprh", GPR32>;
460 def LDAPRW : RCPCLoad<0b10, "ldapr", GPR32>;
461 def LDAPRX : RCPCLoad<0b11, "ldapr", GPR64>;
464 // v8.3a complex add and multiply-accumulate. No predicate here, that is done
465 // inside the multiclass as the FP16 versions need different predicates.
466 defm FCMLA : SIMDThreeSameVectorTiedComplexHSD<1, 0b110, complexrotateop,
468 defm FCADD : SIMDThreeSameVectorComplexHSD<1, 0b111, complexrotateopodd,
470 defm FCMLA : SIMDIndexedTiedComplexHSD<1, 0, 1, complexrotateop, "fcmla",
473 let Predicates = [HasV8_3a] in {
474 // v8.3a Pointer Authentication
475 let Uses = [LR], Defs = [LR] in {
476 def PACIAZ : SystemNoOperands<0b000, "paciaz">;
477 def PACIBZ : SystemNoOperands<0b010, "pacibz">;
478 def AUTIAZ : SystemNoOperands<0b100, "autiaz">;
479 def AUTIBZ : SystemNoOperands<0b110, "autibz">;
481 let Uses = [LR, SP], Defs = [LR] in {
482 def PACIASP : SystemNoOperands<0b001, "paciasp">;
483 def PACIBSP : SystemNoOperands<0b011, "pacibsp">;
484 def AUTIASP : SystemNoOperands<0b101, "autiasp">;
485 def AUTIBSP : SystemNoOperands<0b111, "autibsp">;
487 let Uses = [X16, X17], Defs = [X17], CRm = 0b0001 in {
488 def PACIA1716 : SystemNoOperands<0b000, "pacia1716">;
489 def PACIB1716 : SystemNoOperands<0b010, "pacib1716">;
490 def AUTIA1716 : SystemNoOperands<0b100, "autia1716">;
491 def AUTIB1716 : SystemNoOperands<0b110, "autib1716">;
494 let Uses = [LR], Defs = [LR], CRm = 0b0000 in {
495 def XPACLRI : SystemNoOperands<0b111, "xpaclri">;
498 multiclass SignAuth<bits<3> prefix, bits<3> prefix_z, string asm> {
499 def IA : SignAuthOneData<prefix, 0b00, !strconcat(asm, "ia")>;
500 def IB : SignAuthOneData<prefix, 0b01, !strconcat(asm, "ib")>;
501 def DA : SignAuthOneData<prefix, 0b10, !strconcat(asm, "da")>;
502 def DB : SignAuthOneData<prefix, 0b11, !strconcat(asm, "db")>;
503 def IZA : SignAuthZero<prefix_z, 0b00, !strconcat(asm, "iza")>;
504 def DZA : SignAuthZero<prefix_z, 0b10, !strconcat(asm, "dza")>;
505 def IZB : SignAuthZero<prefix_z, 0b01, !strconcat(asm, "izb")>;
506 def DZB : SignAuthZero<prefix_z, 0b11, !strconcat(asm, "dzb")>;
509 defm PAC : SignAuth<0b000, 0b010, "pac">;
510 defm AUT : SignAuth<0b001, 0b011, "aut">;
512 def XPACI : SignAuthZero<0b100, 0b00, "xpaci">;
513 def XPACD : SignAuthZero<0b100, 0b01, "xpacd">;
514 def PACGA : SignAuthTwoOperand<0b1100, "pacga", null_frag>;
516 // Combined Instructions
517 def BRAA : AuthBranchTwoOperands<0, 0, "braa">;
518 def BRAB : AuthBranchTwoOperands<0, 1, "brab">;
519 def BLRAA : AuthBranchTwoOperands<1, 0, "blraa">;
520 def BLRAB : AuthBranchTwoOperands<1, 1, "blrab">;
522 def BRAAZ : AuthOneOperand<0b000, 0, "braaz">;
523 def BRABZ : AuthOneOperand<0b000, 1, "brabz">;
524 def BLRAAZ : AuthOneOperand<0b001, 0, "blraaz">;
525 def BLRABZ : AuthOneOperand<0b001, 1, "blrabz">;
527 let isReturn = 1 in {
528 def RETAA : AuthReturn<0b010, 0, "retaa">;
529 def RETAB : AuthReturn<0b010, 1, "retab">;
530 def ERETAA : AuthReturn<0b100, 0, "eretaa">;
531 def ERETAB : AuthReturn<0b100, 1, "eretab">;
534 defm LDRAA : AuthLoad<0, "ldraa", simm10Scaled>;
535 defm LDRAB : AuthLoad<1, "ldrab", simm10Scaled>;
537 // v8.3a floating point conversion for javascript
538 let Predicates = [HasV8_3a, HasFPARMv8] in
539 def FJCVTZS : BaseFPToIntegerUnscaled<0b01, 0b11, 0b110, FPR64, GPR32,
546 def : InstAlias<"clrex", (CLREX 0xf)>;
547 def : InstAlias<"isb", (ISB 0xf)>;
551 def MSRpstateImm1 : MSRpstateImm0_1;
552 def MSRpstateImm4 : MSRpstateImm0_15;
554 // The thread pointer (on Linux, at least, where this has been implemented) is
556 def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins),
557 [(set GPR64:$dst, AArch64threadpointer)]>, Sched<[WriteSys]>;
559 // The cycle counter PMC register is PMCCNTR_EL0.
560 let Predicates = [HasPerfMon] in
561 def : Pat<(readcyclecounter), (MRS 0xdce8)>;
563 // Generic system instructions
564 def SYSxt : SystemXtI<0, "sys">;
565 def SYSLxt : SystemLXtI<1, "sysl">;
567 def : InstAlias<"sys $op1, $Cn, $Cm, $op2",
568 (SYSxt imm0_7:$op1, sys_cr_op:$Cn,
569 sys_cr_op:$Cm, imm0_7:$op2, XZR)>;
571 //===----------------------------------------------------------------------===//
572 // Move immediate instructions.
573 //===----------------------------------------------------------------------===//
575 defm MOVK : InsertImmediate<0b11, "movk">;
576 defm MOVN : MoveImmediate<0b00, "movn">;
578 let PostEncoderMethod = "fixMOVZ" in
579 defm MOVZ : MoveImmediate<0b10, "movz">;
581 // First group of aliases covers an implicit "lsl #0".
582 def : InstAlias<"movk $dst, $imm", (MOVKWi GPR32:$dst, imm0_65535:$imm, 0), 0>;
583 def : InstAlias<"movk $dst, $imm", (MOVKXi GPR64:$dst, imm0_65535:$imm, 0), 0>;
584 def : InstAlias<"movn $dst, $imm", (MOVNWi GPR32:$dst, imm0_65535:$imm, 0)>;
585 def : InstAlias<"movn $dst, $imm", (MOVNXi GPR64:$dst, imm0_65535:$imm, 0)>;
586 def : InstAlias<"movz $dst, $imm", (MOVZWi GPR32:$dst, imm0_65535:$imm, 0)>;
587 def : InstAlias<"movz $dst, $imm", (MOVZXi GPR64:$dst, imm0_65535:$imm, 0)>;
589 // Next, we have various ELF relocations with the ":XYZ_g0:sym" syntax.
590 def : InstAlias<"movz $Rd, $sym", (MOVZXi GPR64:$Rd, movz_symbol_g3:$sym, 48)>;
591 def : InstAlias<"movz $Rd, $sym", (MOVZXi GPR64:$Rd, movz_symbol_g2:$sym, 32)>;
592 def : InstAlias<"movz $Rd, $sym", (MOVZXi GPR64:$Rd, movz_symbol_g1:$sym, 16)>;
593 def : InstAlias<"movz $Rd, $sym", (MOVZXi GPR64:$Rd, movz_symbol_g0:$sym, 0)>;
595 def : InstAlias<"movn $Rd, $sym", (MOVNXi GPR64:$Rd, movz_symbol_g3:$sym, 48)>;
596 def : InstAlias<"movn $Rd, $sym", (MOVNXi GPR64:$Rd, movz_symbol_g2:$sym, 32)>;
597 def : InstAlias<"movn $Rd, $sym", (MOVNXi GPR64:$Rd, movz_symbol_g1:$sym, 16)>;
598 def : InstAlias<"movn $Rd, $sym", (MOVNXi GPR64:$Rd, movz_symbol_g0:$sym, 0)>;
600 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movk_symbol_g3:$sym, 48), 0>;
601 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movk_symbol_g2:$sym, 32), 0>;
602 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movk_symbol_g1:$sym, 16), 0>;
603 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movk_symbol_g0:$sym, 0), 0>;
605 def : InstAlias<"movz $Rd, $sym", (MOVZWi GPR32:$Rd, movz_symbol_g1:$sym, 16)>;
606 def : InstAlias<"movz $Rd, $sym", (MOVZWi GPR32:$Rd, movz_symbol_g0:$sym, 0)>;
608 def : InstAlias<"movn $Rd, $sym", (MOVNWi GPR32:$Rd, movz_symbol_g1:$sym, 16)>;
609 def : InstAlias<"movn $Rd, $sym", (MOVNWi GPR32:$Rd, movz_symbol_g0:$sym, 0)>;
611 def : InstAlias<"movk $Rd, $sym", (MOVKWi GPR32:$Rd, movk_symbol_g1:$sym, 16), 0>;
612 def : InstAlias<"movk $Rd, $sym", (MOVKWi GPR32:$Rd, movk_symbol_g0:$sym, 0), 0>;
614 // Final group of aliases covers true "mov $Rd, $imm" cases.
615 multiclass movw_mov_alias<string basename,Instruction INST, RegisterClass GPR,
616 int width, int shift> {
617 def _asmoperand : AsmOperandClass {
618 let Name = basename # width # "_lsl" # shift # "MovAlias";
619 let PredicateMethod = "is" # basename # "MovAlias<" # width # ", "
621 let RenderMethod = "add" # basename # "MovAliasOperands<" # shift # ">";
624 def _movimm : Operand<i32> {
625 let ParserMatchClass = !cast<AsmOperandClass>(NAME # "_asmoperand");
628 def : InstAlias<"mov $Rd, $imm",
629 (INST GPR:$Rd, !cast<Operand>(NAME # "_movimm"):$imm, shift)>;
632 defm : movw_mov_alias<"MOVZ", MOVZWi, GPR32, 32, 0>;
633 defm : movw_mov_alias<"MOVZ", MOVZWi, GPR32, 32, 16>;
635 defm : movw_mov_alias<"MOVZ", MOVZXi, GPR64, 64, 0>;
636 defm : movw_mov_alias<"MOVZ", MOVZXi, GPR64, 64, 16>;
637 defm : movw_mov_alias<"MOVZ", MOVZXi, GPR64, 64, 32>;
638 defm : movw_mov_alias<"MOVZ", MOVZXi, GPR64, 64, 48>;
640 defm : movw_mov_alias<"MOVN", MOVNWi, GPR32, 32, 0>;
641 defm : movw_mov_alias<"MOVN", MOVNWi, GPR32, 32, 16>;
643 defm : movw_mov_alias<"MOVN", MOVNXi, GPR64, 64, 0>;
644 defm : movw_mov_alias<"MOVN", MOVNXi, GPR64, 64, 16>;
645 defm : movw_mov_alias<"MOVN", MOVNXi, GPR64, 64, 32>;
646 defm : movw_mov_alias<"MOVN", MOVNXi, GPR64, 64, 48>;
648 let isReMaterializable = 1, isCodeGenOnly = 1, isMoveImm = 1,
649 isAsCheapAsAMove = 1 in {
650 // FIXME: The following pseudo instructions are only needed because remat
651 // cannot handle multiple instructions. When that changes, we can select
652 // directly to the real instructions and get rid of these pseudos.
655 : Pseudo<(outs GPR32:$dst), (ins i32imm:$src),
656 [(set GPR32:$dst, imm:$src)]>,
659 : Pseudo<(outs GPR64:$dst), (ins i64imm:$src),
660 [(set GPR64:$dst, imm:$src)]>,
662 } // isReMaterializable, isCodeGenOnly
664 // If possible, we want to use MOVi32imm even for 64-bit moves. This gives the
665 // eventual expansion code fewer bits to worry about getting right. Marshalling
666 // the types is a little tricky though:
667 def i64imm_32bit : ImmLeaf<i64, [{
668 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm);
671 def s64imm_32bit : ImmLeaf<i64, [{
672 int64_t Imm64 = static_cast<int64_t>(Imm);
673 return Imm64 >= std::numeric_limits<int32_t>::min() &&
674 Imm64 <= std::numeric_limits<int32_t>::max();
677 def trunc_imm : SDNodeXForm<imm, [{
678 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i32);
681 def : Pat<(i64 i64imm_32bit:$src),
682 (SUBREG_TO_REG (i64 0), (MOVi32imm (trunc_imm imm:$src)), sub_32)>;
684 // Materialize FP constants via MOVi32imm/MOVi64imm (MachO large code model).
685 def bitcast_fpimm_to_i32 : SDNodeXForm<fpimm, [{
686 return CurDAG->getTargetConstant(
687 N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32);
690 def bitcast_fpimm_to_i64 : SDNodeXForm<fpimm, [{
691 return CurDAG->getTargetConstant(
692 N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i64);
696 def : Pat<(f32 fpimm:$in),
697 (COPY_TO_REGCLASS (MOVi32imm (bitcast_fpimm_to_i32 f32:$in)), FPR32)>;
698 def : Pat<(f64 fpimm:$in),
699 (COPY_TO_REGCLASS (MOVi64imm (bitcast_fpimm_to_i64 f64:$in)), FPR64)>;
702 // Deal with the various forms of (ELF) large addressing with MOVZ/MOVK
704 def : Pat<(AArch64WrapperLarge tglobaladdr:$g3, tglobaladdr:$g2,
705 tglobaladdr:$g1, tglobaladdr:$g0),
706 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tglobaladdr:$g0, 0),
707 tglobaladdr:$g1, 16),
708 tglobaladdr:$g2, 32),
709 tglobaladdr:$g3, 48)>;
711 def : Pat<(AArch64WrapperLarge tblockaddress:$g3, tblockaddress:$g2,
712 tblockaddress:$g1, tblockaddress:$g0),
713 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tblockaddress:$g0, 0),
714 tblockaddress:$g1, 16),
715 tblockaddress:$g2, 32),
716 tblockaddress:$g3, 48)>;
718 def : Pat<(AArch64WrapperLarge tconstpool:$g3, tconstpool:$g2,
719 tconstpool:$g1, tconstpool:$g0),
720 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tconstpool:$g0, 0),
723 tconstpool:$g3, 48)>;
725 def : Pat<(AArch64WrapperLarge tjumptable:$g3, tjumptable:$g2,
726 tjumptable:$g1, tjumptable:$g0),
727 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tjumptable:$g0, 0),
730 tjumptable:$g3, 48)>;
733 //===----------------------------------------------------------------------===//
734 // Arithmetic instructions.
735 //===----------------------------------------------------------------------===//
737 // Add/subtract with carry.
738 defm ADC : AddSubCarry<0, "adc", "adcs", AArch64adc, AArch64adc_flag>;
739 defm SBC : AddSubCarry<1, "sbc", "sbcs", AArch64sbc, AArch64sbc_flag>;
741 def : InstAlias<"ngc $dst, $src", (SBCWr GPR32:$dst, WZR, GPR32:$src)>;
742 def : InstAlias<"ngc $dst, $src", (SBCXr GPR64:$dst, XZR, GPR64:$src)>;
743 def : InstAlias<"ngcs $dst, $src", (SBCSWr GPR32:$dst, WZR, GPR32:$src)>;
744 def : InstAlias<"ngcs $dst, $src", (SBCSXr GPR64:$dst, XZR, GPR64:$src)>;
747 defm ADD : AddSub<0, "add", "sub", add>;
748 defm SUB : AddSub<1, "sub", "add">;
750 def : InstAlias<"mov $dst, $src",
751 (ADDWri GPR32sponly:$dst, GPR32sp:$src, 0, 0)>;
752 def : InstAlias<"mov $dst, $src",
753 (ADDWri GPR32sp:$dst, GPR32sponly:$src, 0, 0)>;
754 def : InstAlias<"mov $dst, $src",
755 (ADDXri GPR64sponly:$dst, GPR64sp:$src, 0, 0)>;
756 def : InstAlias<"mov $dst, $src",
757 (ADDXri GPR64sp:$dst, GPR64sponly:$src, 0, 0)>;
759 defm ADDS : AddSubS<0, "adds", AArch64add_flag, "cmn", "subs", "cmp">;
760 defm SUBS : AddSubS<1, "subs", AArch64sub_flag, "cmp", "adds", "cmn">;
762 // Use SUBS instead of SUB to enable CSE between SUBS and SUB.
763 def : Pat<(sub GPR32sp:$Rn, addsub_shifted_imm32:$imm),
764 (SUBSWri GPR32sp:$Rn, addsub_shifted_imm32:$imm)>;
765 def : Pat<(sub GPR64sp:$Rn, addsub_shifted_imm64:$imm),
766 (SUBSXri GPR64sp:$Rn, addsub_shifted_imm64:$imm)>;
767 def : Pat<(sub GPR32:$Rn, GPR32:$Rm),
768 (SUBSWrr GPR32:$Rn, GPR32:$Rm)>;
769 def : Pat<(sub GPR64:$Rn, GPR64:$Rm),
770 (SUBSXrr GPR64:$Rn, GPR64:$Rm)>;
771 def : Pat<(sub GPR32:$Rn, arith_shifted_reg32:$Rm),
772 (SUBSWrs GPR32:$Rn, arith_shifted_reg32:$Rm)>;
773 def : Pat<(sub GPR64:$Rn, arith_shifted_reg64:$Rm),
774 (SUBSXrs GPR64:$Rn, arith_shifted_reg64:$Rm)>;
775 let AddedComplexity = 1 in {
776 def : Pat<(sub GPR32sp:$R2, arith_extended_reg32<i32>:$R3),
777 (SUBSWrx GPR32sp:$R2, arith_extended_reg32<i32>:$R3)>;
778 def : Pat<(sub GPR64sp:$R2, arith_extended_reg32to64<i64>:$R3),
779 (SUBSXrx GPR64sp:$R2, arith_extended_reg32to64<i64>:$R3)>;
782 // Because of the immediate format for add/sub-imm instructions, the
783 // expression (add x, -1) must be transformed to (SUB{W,X}ri x, 1).
784 // These patterns capture that transformation.
785 let AddedComplexity = 1 in {
786 def : Pat<(add GPR32:$Rn, neg_addsub_shifted_imm32:$imm),
787 (SUBSWri GPR32:$Rn, neg_addsub_shifted_imm32:$imm)>;
788 def : Pat<(add GPR64:$Rn, neg_addsub_shifted_imm64:$imm),
789 (SUBSXri GPR64:$Rn, neg_addsub_shifted_imm64:$imm)>;
790 def : Pat<(sub GPR32:$Rn, neg_addsub_shifted_imm32:$imm),
791 (ADDWri GPR32:$Rn, neg_addsub_shifted_imm32:$imm)>;
792 def : Pat<(sub GPR64:$Rn, neg_addsub_shifted_imm64:$imm),
793 (ADDXri GPR64:$Rn, neg_addsub_shifted_imm64:$imm)>;
796 // Because of the immediate format for add/sub-imm instructions, the
797 // expression (add x, -1) must be transformed to (SUB{W,X}ri x, 1).
798 // These patterns capture that transformation.
799 let AddedComplexity = 1 in {
800 def : Pat<(AArch64add_flag GPR32:$Rn, neg_addsub_shifted_imm32:$imm),
801 (SUBSWri GPR32:$Rn, neg_addsub_shifted_imm32:$imm)>;
802 def : Pat<(AArch64add_flag GPR64:$Rn, neg_addsub_shifted_imm64:$imm),
803 (SUBSXri GPR64:$Rn, neg_addsub_shifted_imm64:$imm)>;
804 def : Pat<(AArch64sub_flag GPR32:$Rn, neg_addsub_shifted_imm32:$imm),
805 (ADDSWri GPR32:$Rn, neg_addsub_shifted_imm32:$imm)>;
806 def : Pat<(AArch64sub_flag GPR64:$Rn, neg_addsub_shifted_imm64:$imm),
807 (ADDSXri GPR64:$Rn, neg_addsub_shifted_imm64:$imm)>;
810 def : InstAlias<"neg $dst, $src", (SUBWrs GPR32:$dst, WZR, GPR32:$src, 0), 3>;
811 def : InstAlias<"neg $dst, $src", (SUBXrs GPR64:$dst, XZR, GPR64:$src, 0), 3>;
812 def : InstAlias<"neg $dst, $src$shift",
813 (SUBWrs GPR32:$dst, WZR, GPR32:$src, arith_shift32:$shift), 2>;
814 def : InstAlias<"neg $dst, $src$shift",
815 (SUBXrs GPR64:$dst, XZR, GPR64:$src, arith_shift64:$shift), 2>;
817 def : InstAlias<"negs $dst, $src", (SUBSWrs GPR32:$dst, WZR, GPR32:$src, 0), 3>;
818 def : InstAlias<"negs $dst, $src", (SUBSXrs GPR64:$dst, XZR, GPR64:$src, 0), 3>;
819 def : InstAlias<"negs $dst, $src$shift",
820 (SUBSWrs GPR32:$dst, WZR, GPR32:$src, arith_shift32:$shift), 2>;
821 def : InstAlias<"negs $dst, $src$shift",
822 (SUBSXrs GPR64:$dst, XZR, GPR64:$src, arith_shift64:$shift), 2>;
825 // Unsigned/Signed divide
826 defm UDIV : Div<0, "udiv", udiv>;
827 defm SDIV : Div<1, "sdiv", sdiv>;
829 def : Pat<(int_aarch64_udiv GPR32:$Rn, GPR32:$Rm), (UDIVWr GPR32:$Rn, GPR32:$Rm)>;
830 def : Pat<(int_aarch64_udiv GPR64:$Rn, GPR64:$Rm), (UDIVXr GPR64:$Rn, GPR64:$Rm)>;
831 def : Pat<(int_aarch64_sdiv GPR32:$Rn, GPR32:$Rm), (SDIVWr GPR32:$Rn, GPR32:$Rm)>;
832 def : Pat<(int_aarch64_sdiv GPR64:$Rn, GPR64:$Rm), (SDIVXr GPR64:$Rn, GPR64:$Rm)>;
835 defm ASRV : Shift<0b10, "asr", sra>;
836 defm LSLV : Shift<0b00, "lsl", shl>;
837 defm LSRV : Shift<0b01, "lsr", srl>;
838 defm RORV : Shift<0b11, "ror", rotr>;
840 def : ShiftAlias<"asrv", ASRVWr, GPR32>;
841 def : ShiftAlias<"asrv", ASRVXr, GPR64>;
842 def : ShiftAlias<"lslv", LSLVWr, GPR32>;
843 def : ShiftAlias<"lslv", LSLVXr, GPR64>;
844 def : ShiftAlias<"lsrv", LSRVWr, GPR32>;
845 def : ShiftAlias<"lsrv", LSRVXr, GPR64>;
846 def : ShiftAlias<"rorv", RORVWr, GPR32>;
847 def : ShiftAlias<"rorv", RORVXr, GPR64>;
850 let AddedComplexity = 5 in {
851 defm MADD : MulAccum<0, "madd", add>;
852 defm MSUB : MulAccum<1, "msub", sub>;
854 def : Pat<(i32 (mul GPR32:$Rn, GPR32:$Rm)),
855 (MADDWrrr GPR32:$Rn, GPR32:$Rm, WZR)>;
856 def : Pat<(i64 (mul GPR64:$Rn, GPR64:$Rm)),
857 (MADDXrrr GPR64:$Rn, GPR64:$Rm, XZR)>;
859 def : Pat<(i32 (ineg (mul GPR32:$Rn, GPR32:$Rm))),
860 (MSUBWrrr GPR32:$Rn, GPR32:$Rm, WZR)>;
861 def : Pat<(i64 (ineg (mul GPR64:$Rn, GPR64:$Rm))),
862 (MSUBXrrr GPR64:$Rn, GPR64:$Rm, XZR)>;
863 def : Pat<(i32 (mul (ineg GPR32:$Rn), GPR32:$Rm)),
864 (MSUBWrrr GPR32:$Rn, GPR32:$Rm, WZR)>;
865 def : Pat<(i64 (mul (ineg GPR64:$Rn), GPR64:$Rm)),
866 (MSUBXrrr GPR64:$Rn, GPR64:$Rm, XZR)>;
867 } // AddedComplexity = 5
869 let AddedComplexity = 5 in {
870 def SMADDLrrr : WideMulAccum<0, 0b001, "smaddl", add, sext>;
871 def SMSUBLrrr : WideMulAccum<1, 0b001, "smsubl", sub, sext>;
872 def UMADDLrrr : WideMulAccum<0, 0b101, "umaddl", add, zext>;
873 def UMSUBLrrr : WideMulAccum<1, 0b101, "umsubl", sub, zext>;
875 def : Pat<(i64 (mul (sext GPR32:$Rn), (sext GPR32:$Rm))),
876 (SMADDLrrr GPR32:$Rn, GPR32:$Rm, XZR)>;
877 def : Pat<(i64 (mul (zext GPR32:$Rn), (zext GPR32:$Rm))),
878 (UMADDLrrr GPR32:$Rn, GPR32:$Rm, XZR)>;
880 def : Pat<(i64 (ineg (mul (sext GPR32:$Rn), (sext GPR32:$Rm)))),
881 (SMSUBLrrr GPR32:$Rn, GPR32:$Rm, XZR)>;
882 def : Pat<(i64 (ineg (mul (zext GPR32:$Rn), (zext GPR32:$Rm)))),
883 (UMSUBLrrr GPR32:$Rn, GPR32:$Rm, XZR)>;
885 def : Pat<(i64 (mul (sext GPR32:$Rn), (s64imm_32bit:$C))),
886 (SMADDLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), XZR)>;
887 def : Pat<(i64 (mul (zext GPR32:$Rn), (i64imm_32bit:$C))),
888 (UMADDLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), XZR)>;
889 def : Pat<(i64 (mul (sext_inreg GPR64:$Rn, i32), (s64imm_32bit:$C))),
890 (SMADDLrrr (i32 (EXTRACT_SUBREG GPR64:$Rn, sub_32)),
891 (MOVi32imm (trunc_imm imm:$C)), XZR)>;
893 def : Pat<(i64 (ineg (mul (sext GPR32:$Rn), (s64imm_32bit:$C)))),
894 (SMSUBLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), XZR)>;
895 def : Pat<(i64 (ineg (mul (zext GPR32:$Rn), (i64imm_32bit:$C)))),
896 (UMSUBLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), XZR)>;
897 def : Pat<(i64 (ineg (mul (sext_inreg GPR64:$Rn, i32), (s64imm_32bit:$C)))),
898 (SMSUBLrrr (i32 (EXTRACT_SUBREG GPR64:$Rn, sub_32)),
899 (MOVi32imm (trunc_imm imm:$C)), XZR)>;
901 def : Pat<(i64 (add (mul (sext GPR32:$Rn), (s64imm_32bit:$C)), GPR64:$Ra)),
902 (SMADDLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), GPR64:$Ra)>;
903 def : Pat<(i64 (add (mul (zext GPR32:$Rn), (i64imm_32bit:$C)), GPR64:$Ra)),
904 (UMADDLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), GPR64:$Ra)>;
905 def : Pat<(i64 (add (mul (sext_inreg GPR64:$Rn, i32), (s64imm_32bit:$C)),
907 (SMADDLrrr (i32 (EXTRACT_SUBREG GPR64:$Rn, sub_32)),
908 (MOVi32imm (trunc_imm imm:$C)), GPR64:$Ra)>;
910 def : Pat<(i64 (sub GPR64:$Ra, (mul (sext GPR32:$Rn), (s64imm_32bit:$C)))),
911 (SMSUBLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), GPR64:$Ra)>;
912 def : Pat<(i64 (sub GPR64:$Ra, (mul (zext GPR32:$Rn), (i64imm_32bit:$C)))),
913 (UMSUBLrrr GPR32:$Rn, (MOVi32imm (trunc_imm imm:$C)), GPR64:$Ra)>;
914 def : Pat<(i64 (sub GPR64:$Ra, (mul (sext_inreg GPR64:$Rn, i32),
915 (s64imm_32bit:$C)))),
916 (SMSUBLrrr (i32 (EXTRACT_SUBREG GPR64:$Rn, sub_32)),
917 (MOVi32imm (trunc_imm imm:$C)), GPR64:$Ra)>;
918 } // AddedComplexity = 5
920 def : MulAccumWAlias<"mul", MADDWrrr>;
921 def : MulAccumXAlias<"mul", MADDXrrr>;
922 def : MulAccumWAlias<"mneg", MSUBWrrr>;
923 def : MulAccumXAlias<"mneg", MSUBXrrr>;
924 def : WideMulAccumAlias<"smull", SMADDLrrr>;
925 def : WideMulAccumAlias<"smnegl", SMSUBLrrr>;
926 def : WideMulAccumAlias<"umull", UMADDLrrr>;
927 def : WideMulAccumAlias<"umnegl", UMSUBLrrr>;
930 def SMULHrr : MulHi<0b010, "smulh", mulhs>;
931 def UMULHrr : MulHi<0b110, "umulh", mulhu>;
934 def CRC32Brr : BaseCRC32<0, 0b00, 0, GPR32, int_aarch64_crc32b, "crc32b">;
935 def CRC32Hrr : BaseCRC32<0, 0b01, 0, GPR32, int_aarch64_crc32h, "crc32h">;
936 def CRC32Wrr : BaseCRC32<0, 0b10, 0, GPR32, int_aarch64_crc32w, "crc32w">;
937 def CRC32Xrr : BaseCRC32<1, 0b11, 0, GPR64, int_aarch64_crc32x, "crc32x">;
939 def CRC32CBrr : BaseCRC32<0, 0b00, 1, GPR32, int_aarch64_crc32cb, "crc32cb">;
940 def CRC32CHrr : BaseCRC32<0, 0b01, 1, GPR32, int_aarch64_crc32ch, "crc32ch">;
941 def CRC32CWrr : BaseCRC32<0, 0b10, 1, GPR32, int_aarch64_crc32cw, "crc32cw">;
942 def CRC32CXrr : BaseCRC32<1, 0b11, 1, GPR64, int_aarch64_crc32cx, "crc32cx">;
945 defm CAS : CompareAndSwap<0, 0, "">;
946 defm CASA : CompareAndSwap<1, 0, "a">;
947 defm CASL : CompareAndSwap<0, 1, "l">;
948 defm CASAL : CompareAndSwap<1, 1, "al">;
951 defm CASP : CompareAndSwapPair<0, 0, "">;
952 defm CASPA : CompareAndSwapPair<1, 0, "a">;
953 defm CASPL : CompareAndSwapPair<0, 1, "l">;
954 defm CASPAL : CompareAndSwapPair<1, 1, "al">;
957 defm SWP : Swap<0, 0, "">;
958 defm SWPA : Swap<1, 0, "a">;
959 defm SWPL : Swap<0, 1, "l">;
960 defm SWPAL : Swap<1, 1, "al">;
962 // v8.1 atomic LD<OP>(register). Performs load and then ST<OP>(register)
963 defm LDADD : LDOPregister<0b000, "add", 0, 0, "">;
964 defm LDADDA : LDOPregister<0b000, "add", 1, 0, "a">;
965 defm LDADDL : LDOPregister<0b000, "add", 0, 1, "l">;
966 defm LDADDAL : LDOPregister<0b000, "add", 1, 1, "al">;
968 defm LDCLR : LDOPregister<0b001, "clr", 0, 0, "">;
969 defm LDCLRA : LDOPregister<0b001, "clr", 1, 0, "a">;
970 defm LDCLRL : LDOPregister<0b001, "clr", 0, 1, "l">;
971 defm LDCLRAL : LDOPregister<0b001, "clr", 1, 1, "al">;
973 defm LDEOR : LDOPregister<0b010, "eor", 0, 0, "">;
974 defm LDEORA : LDOPregister<0b010, "eor", 1, 0, "a">;
975 defm LDEORL : LDOPregister<0b010, "eor", 0, 1, "l">;
976 defm LDEORAL : LDOPregister<0b010, "eor", 1, 1, "al">;
978 defm LDSET : LDOPregister<0b011, "set", 0, 0, "">;
979 defm LDSETA : LDOPregister<0b011, "set", 1, 0, "a">;
980 defm LDSETL : LDOPregister<0b011, "set", 0, 1, "l">;
981 defm LDSETAL : LDOPregister<0b011, "set", 1, 1, "al">;
983 defm LDSMAX : LDOPregister<0b100, "smax", 0, 0, "">;
984 defm LDSMAXA : LDOPregister<0b100, "smax", 1, 0, "a">;
985 defm LDSMAXL : LDOPregister<0b100, "smax", 0, 1, "l">;
986 defm LDSMAXAL : LDOPregister<0b100, "smax", 1, 1, "al">;
988 defm LDSMIN : LDOPregister<0b101, "smin", 0, 0, "">;
989 defm LDSMINA : LDOPregister<0b101, "smin", 1, 0, "a">;
990 defm LDSMINL : LDOPregister<0b101, "smin", 0, 1, "l">;
991 defm LDSMINAL : LDOPregister<0b101, "smin", 1, 1, "al">;
993 defm LDUMAX : LDOPregister<0b110, "umax", 0, 0, "">;
994 defm LDUMAXA : LDOPregister<0b110, "umax", 1, 0, "a">;
995 defm LDUMAXL : LDOPregister<0b110, "umax", 0, 1, "l">;
996 defm LDUMAXAL : LDOPregister<0b110, "umax", 1, 1, "al">;
998 defm LDUMIN : LDOPregister<0b111, "umin", 0, 0, "">;
999 defm LDUMINA : LDOPregister<0b111, "umin", 1, 0, "a">;
1000 defm LDUMINL : LDOPregister<0b111, "umin", 0, 1, "l">;
1001 defm LDUMINAL : LDOPregister<0b111, "umin", 1, 1, "al">;
1003 // v8.1 atomic ST<OP>(register) as aliases to "LD<OP>(register) when Rt=xZR"
1004 defm : STOPregister<"stadd","LDADD">; // STADDx
1005 defm : STOPregister<"stclr","LDCLR">; // STCLRx
1006 defm : STOPregister<"steor","LDEOR">; // STEORx
1007 defm : STOPregister<"stset","LDSET">; // STSETx
1008 defm : STOPregister<"stsmax","LDSMAX">;// STSMAXx
1009 defm : STOPregister<"stsmin","LDSMIN">;// STSMINx
1010 defm : STOPregister<"stumax","LDUMAX">;// STUMAXx
1011 defm : STOPregister<"stumin","LDUMIN">;// STUMINx
1013 //===----------------------------------------------------------------------===//
1014 // Logical instructions.
1015 //===----------------------------------------------------------------------===//
1018 defm ANDS : LogicalImmS<0b11, "ands", AArch64and_flag, "bics">;
1019 defm AND : LogicalImm<0b00, "and", and, "bic">;
1020 defm EOR : LogicalImm<0b10, "eor", xor, "eon">;
1021 defm ORR : LogicalImm<0b01, "orr", or, "orn">;
1023 // FIXME: these aliases *are* canonical sometimes (when movz can't be
1024 // used). Actually, it seems to be working right now, but putting logical_immXX
1025 // here is a bit dodgy on the AsmParser side too.
1026 def : InstAlias<"mov $dst, $imm", (ORRWri GPR32sp:$dst, WZR,
1027 logical_imm32:$imm), 0>;
1028 def : InstAlias<"mov $dst, $imm", (ORRXri GPR64sp:$dst, XZR,
1029 logical_imm64:$imm), 0>;
1033 defm ANDS : LogicalRegS<0b11, 0, "ands", AArch64and_flag>;
1034 defm BICS : LogicalRegS<0b11, 1, "bics",
1035 BinOpFrag<(AArch64and_flag node:$LHS, (not node:$RHS))>>;
1036 defm AND : LogicalReg<0b00, 0, "and", and>;
1037 defm BIC : LogicalReg<0b00, 1, "bic",
1038 BinOpFrag<(and node:$LHS, (not node:$RHS))>>;
1039 defm EON : LogicalReg<0b10, 1, "eon",
1040 BinOpFrag<(not (xor node:$LHS, node:$RHS))>>;
1041 defm EOR : LogicalReg<0b10, 0, "eor", xor>;
1042 defm ORN : LogicalReg<0b01, 1, "orn",
1043 BinOpFrag<(or node:$LHS, (not node:$RHS))>>;
1044 defm ORR : LogicalReg<0b01, 0, "orr", or>;
1046 def : InstAlias<"mov $dst, $src", (ORRWrs GPR32:$dst, WZR, GPR32:$src, 0), 2>;
1047 def : InstAlias<"mov $dst, $src", (ORRXrs GPR64:$dst, XZR, GPR64:$src, 0), 2>;
1049 def : InstAlias<"mvn $Wd, $Wm", (ORNWrs GPR32:$Wd, WZR, GPR32:$Wm, 0), 3>;
1050 def : InstAlias<"mvn $Xd, $Xm", (ORNXrs GPR64:$Xd, XZR, GPR64:$Xm, 0), 3>;
1052 def : InstAlias<"mvn $Wd, $Wm$sh",
1053 (ORNWrs GPR32:$Wd, WZR, GPR32:$Wm, logical_shift32:$sh), 2>;
1054 def : InstAlias<"mvn $Xd, $Xm$sh",
1055 (ORNXrs GPR64:$Xd, XZR, GPR64:$Xm, logical_shift64:$sh), 2>;
1057 def : InstAlias<"tst $src1, $src2",
1058 (ANDSWri WZR, GPR32:$src1, logical_imm32:$src2), 2>;
1059 def : InstAlias<"tst $src1, $src2",
1060 (ANDSXri XZR, GPR64:$src1, logical_imm64:$src2), 2>;
1062 def : InstAlias<"tst $src1, $src2",
1063 (ANDSWrs WZR, GPR32:$src1, GPR32:$src2, 0), 3>;
1064 def : InstAlias<"tst $src1, $src2",
1065 (ANDSXrs XZR, GPR64:$src1, GPR64:$src2, 0), 3>;
1067 def : InstAlias<"tst $src1, $src2$sh",
1068 (ANDSWrs WZR, GPR32:$src1, GPR32:$src2, logical_shift32:$sh), 2>;
1069 def : InstAlias<"tst $src1, $src2$sh",
1070 (ANDSXrs XZR, GPR64:$src1, GPR64:$src2, logical_shift64:$sh), 2>;
1073 def : Pat<(not GPR32:$Wm), (ORNWrr WZR, GPR32:$Wm)>;
1074 def : Pat<(not GPR64:$Xm), (ORNXrr XZR, GPR64:$Xm)>;
1077 //===----------------------------------------------------------------------===//
1078 // One operand data processing instructions.
1079 //===----------------------------------------------------------------------===//
1081 defm CLS : OneOperandData<0b101, "cls">;
1082 defm CLZ : OneOperandData<0b100, "clz", ctlz>;
1083 defm RBIT : OneOperandData<0b000, "rbit", bitreverse>;
1085 def REV16Wr : OneWRegData<0b001, "rev16",
1086 UnOpFrag<(rotr (bswap node:$LHS), (i64 16))>>;
1087 def REV16Xr : OneXRegData<0b001, "rev16", null_frag>;
1089 def : Pat<(cttz GPR32:$Rn),
1090 (CLZWr (RBITWr GPR32:$Rn))>;
1091 def : Pat<(cttz GPR64:$Rn),
1092 (CLZXr (RBITXr GPR64:$Rn))>;
1093 def : Pat<(ctlz (or (shl (xor (sra GPR32:$Rn, (i64 31)), GPR32:$Rn), (i64 1)),
1096 def : Pat<(ctlz (or (shl (xor (sra GPR64:$Rn, (i64 63)), GPR64:$Rn), (i64 1)),
1100 // Unlike the other one operand instructions, the instructions with the "rev"
1101 // mnemonic do *not* just different in the size bit, but actually use different
1102 // opcode bits for the different sizes.
1103 def REVWr : OneWRegData<0b010, "rev", bswap>;
1104 def REVXr : OneXRegData<0b011, "rev", bswap>;
1105 def REV32Xr : OneXRegData<0b010, "rev32",
1106 UnOpFrag<(rotr (bswap node:$LHS), (i64 32))>>;
1108 def : InstAlias<"rev64 $Rd, $Rn", (REVXr GPR64:$Rd, GPR64:$Rn), 0>;
1110 // The bswap commutes with the rotr so we want a pattern for both possible
1112 def : Pat<(bswap (rotr GPR32:$Rn, (i64 16))), (REV16Wr GPR32:$Rn)>;
1113 def : Pat<(bswap (rotr GPR64:$Rn, (i64 32))), (REV32Xr GPR64:$Rn)>;
1115 //===----------------------------------------------------------------------===//
1116 // Bitfield immediate extraction instruction.
1117 //===----------------------------------------------------------------------===//
1118 let hasSideEffects = 0 in
1119 defm EXTR : ExtractImm<"extr">;
1120 def : InstAlias<"ror $dst, $src, $shift",
1121 (EXTRWrri GPR32:$dst, GPR32:$src, GPR32:$src, imm0_31:$shift)>;
1122 def : InstAlias<"ror $dst, $src, $shift",
1123 (EXTRXrri GPR64:$dst, GPR64:$src, GPR64:$src, imm0_63:$shift)>;
1125 def : Pat<(rotr GPR32:$Rn, (i64 imm0_31:$imm)),
1126 (EXTRWrri GPR32:$Rn, GPR32:$Rn, imm0_31:$imm)>;
1127 def : Pat<(rotr GPR64:$Rn, (i64 imm0_63:$imm)),
1128 (EXTRXrri GPR64:$Rn, GPR64:$Rn, imm0_63:$imm)>;
1130 //===----------------------------------------------------------------------===//
1131 // Other bitfield immediate instructions.
1132 //===----------------------------------------------------------------------===//
1133 let hasSideEffects = 0 in {
1134 defm BFM : BitfieldImmWith2RegArgs<0b01, "bfm">;
1135 defm SBFM : BitfieldImm<0b00, "sbfm">;
1136 defm UBFM : BitfieldImm<0b10, "ubfm">;
1139 def i32shift_a : Operand<i64>, SDNodeXForm<imm, [{
1140 uint64_t enc = (32 - N->getZExtValue()) & 0x1f;
1141 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1144 def i32shift_b : Operand<i64>, SDNodeXForm<imm, [{
1145 uint64_t enc = 31 - N->getZExtValue();
1146 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1149 // min(7, 31 - shift_amt)
1150 def i32shift_sext_i8 : Operand<i64>, SDNodeXForm<imm, [{
1151 uint64_t enc = 31 - N->getZExtValue();
1152 enc = enc > 7 ? 7 : enc;
1153 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1156 // min(15, 31 - shift_amt)
1157 def i32shift_sext_i16 : Operand<i64>, SDNodeXForm<imm, [{
1158 uint64_t enc = 31 - N->getZExtValue();
1159 enc = enc > 15 ? 15 : enc;
1160 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1163 def i64shift_a : Operand<i64>, SDNodeXForm<imm, [{
1164 uint64_t enc = (64 - N->getZExtValue()) & 0x3f;
1165 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1168 def i64shift_b : Operand<i64>, SDNodeXForm<imm, [{
1169 uint64_t enc = 63 - N->getZExtValue();
1170 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1173 // min(7, 63 - shift_amt)
1174 def i64shift_sext_i8 : Operand<i64>, SDNodeXForm<imm, [{
1175 uint64_t enc = 63 - N->getZExtValue();
1176 enc = enc > 7 ? 7 : enc;
1177 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1180 // min(15, 63 - shift_amt)
1181 def i64shift_sext_i16 : Operand<i64>, SDNodeXForm<imm, [{
1182 uint64_t enc = 63 - N->getZExtValue();
1183 enc = enc > 15 ? 15 : enc;
1184 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1187 // min(31, 63 - shift_amt)
1188 def i64shift_sext_i32 : Operand<i64>, SDNodeXForm<imm, [{
1189 uint64_t enc = 63 - N->getZExtValue();
1190 enc = enc > 31 ? 31 : enc;
1191 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
1194 def : Pat<(shl GPR32:$Rn, (i64 imm0_31:$imm)),
1195 (UBFMWri GPR32:$Rn, (i64 (i32shift_a imm0_31:$imm)),
1196 (i64 (i32shift_b imm0_31:$imm)))>;
1197 def : Pat<(shl GPR64:$Rn, (i64 imm0_63:$imm)),
1198 (UBFMXri GPR64:$Rn, (i64 (i64shift_a imm0_63:$imm)),
1199 (i64 (i64shift_b imm0_63:$imm)))>;
1201 let AddedComplexity = 10 in {
1202 def : Pat<(sra GPR32:$Rn, (i64 imm0_31:$imm)),
1203 (SBFMWri GPR32:$Rn, imm0_31:$imm, 31)>;
1204 def : Pat<(sra GPR64:$Rn, (i64 imm0_63:$imm)),
1205 (SBFMXri GPR64:$Rn, imm0_63:$imm, 63)>;
1208 def : InstAlias<"asr $dst, $src, $shift",
1209 (SBFMWri GPR32:$dst, GPR32:$src, imm0_31:$shift, 31)>;
1210 def : InstAlias<"asr $dst, $src, $shift",
1211 (SBFMXri GPR64:$dst, GPR64:$src, imm0_63:$shift, 63)>;
1212 def : InstAlias<"sxtb $dst, $src", (SBFMWri GPR32:$dst, GPR32:$src, 0, 7)>;
1213 def : InstAlias<"sxtb $dst, $src", (SBFMXri GPR64:$dst, GPR64:$src, 0, 7)>;
1214 def : InstAlias<"sxth $dst, $src", (SBFMWri GPR32:$dst, GPR32:$src, 0, 15)>;
1215 def : InstAlias<"sxth $dst, $src", (SBFMXri GPR64:$dst, GPR64:$src, 0, 15)>;
1216 def : InstAlias<"sxtw $dst, $src", (SBFMXri GPR64:$dst, GPR64:$src, 0, 31)>;
1218 def : Pat<(srl GPR32:$Rn, (i64 imm0_31:$imm)),
1219 (UBFMWri GPR32:$Rn, imm0_31:$imm, 31)>;
1220 def : Pat<(srl GPR64:$Rn, (i64 imm0_63:$imm)),
1221 (UBFMXri GPR64:$Rn, imm0_63:$imm, 63)>;
1223 def : InstAlias<"lsr $dst, $src, $shift",
1224 (UBFMWri GPR32:$dst, GPR32:$src, imm0_31:$shift, 31)>;
1225 def : InstAlias<"lsr $dst, $src, $shift",
1226 (UBFMXri GPR64:$dst, GPR64:$src, imm0_63:$shift, 63)>;
1227 def : InstAlias<"uxtb $dst, $src", (UBFMWri GPR32:$dst, GPR32:$src, 0, 7)>;
1228 def : InstAlias<"uxtb $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 7)>;
1229 def : InstAlias<"uxth $dst, $src", (UBFMWri GPR32:$dst, GPR32:$src, 0, 15)>;
1230 def : InstAlias<"uxth $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 15)>;
1231 def : InstAlias<"uxtw $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 31)>;
1233 //===----------------------------------------------------------------------===//
1234 // Conditional comparison instructions.
1235 //===----------------------------------------------------------------------===//
1236 defm CCMN : CondComparison<0, "ccmn", AArch64ccmn>;
1237 defm CCMP : CondComparison<1, "ccmp", AArch64ccmp>;
1239 //===----------------------------------------------------------------------===//
1240 // Conditional select instructions.
1241 //===----------------------------------------------------------------------===//
1242 defm CSEL : CondSelect<0, 0b00, "csel">;
1244 def inc : PatFrag<(ops node:$in), (add node:$in, 1)>;
1245 defm CSINC : CondSelectOp<0, 0b01, "csinc", inc>;
1246 defm CSINV : CondSelectOp<1, 0b00, "csinv", not>;
1247 defm CSNEG : CondSelectOp<1, 0b01, "csneg", ineg>;
1249 def : Pat<(AArch64csinv GPR32:$tval, GPR32:$fval, (i32 imm:$cc), NZCV),
1250 (CSINVWr GPR32:$tval, GPR32:$fval, (i32 imm:$cc))>;
1251 def : Pat<(AArch64csinv GPR64:$tval, GPR64:$fval, (i32 imm:$cc), NZCV),
1252 (CSINVXr GPR64:$tval, GPR64:$fval, (i32 imm:$cc))>;
1253 def : Pat<(AArch64csneg GPR32:$tval, GPR32:$fval, (i32 imm:$cc), NZCV),
1254 (CSNEGWr GPR32:$tval, GPR32:$fval, (i32 imm:$cc))>;
1255 def : Pat<(AArch64csneg GPR64:$tval, GPR64:$fval, (i32 imm:$cc), NZCV),
1256 (CSNEGXr GPR64:$tval, GPR64:$fval, (i32 imm:$cc))>;
1257 def : Pat<(AArch64csinc GPR32:$tval, GPR32:$fval, (i32 imm:$cc), NZCV),
1258 (CSINCWr GPR32:$tval, GPR32:$fval, (i32 imm:$cc))>;
1259 def : Pat<(AArch64csinc GPR64:$tval, GPR64:$fval, (i32 imm:$cc), NZCV),
1260 (CSINCXr GPR64:$tval, GPR64:$fval, (i32 imm:$cc))>;
1262 def : Pat<(AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV),
1263 (CSINCWr WZR, WZR, (i32 imm:$cc))>;
1264 def : Pat<(AArch64csel (i64 0), (i64 1), (i32 imm:$cc), NZCV),
1265 (CSINCXr XZR, XZR, (i32 imm:$cc))>;
1266 def : Pat<(AArch64csel GPR32:$tval, (i32 1), (i32 imm:$cc), NZCV),
1267 (CSINCWr GPR32:$tval, WZR, (i32 imm:$cc))>;
1268 def : Pat<(AArch64csel GPR64:$tval, (i64 1), (i32 imm:$cc), NZCV),
1269 (CSINCXr GPR64:$tval, XZR, (i32 imm:$cc))>;
1270 def : Pat<(AArch64csel (i32 1), GPR32:$fval, (i32 imm:$cc), NZCV),
1271 (CSINCWr GPR32:$fval, WZR, (i32 (inv_cond_XFORM imm:$cc)))>;
1272 def : Pat<(AArch64csel (i64 1), GPR64:$fval, (i32 imm:$cc), NZCV),
1273 (CSINCXr GPR64:$fval, XZR, (i32 (inv_cond_XFORM imm:$cc)))>;
1274 def : Pat<(AArch64csel (i32 0), (i32 -1), (i32 imm:$cc), NZCV),
1275 (CSINVWr WZR, WZR, (i32 imm:$cc))>;
1276 def : Pat<(AArch64csel (i64 0), (i64 -1), (i32 imm:$cc), NZCV),
1277 (CSINVXr XZR, XZR, (i32 imm:$cc))>;
1278 def : Pat<(AArch64csel GPR32:$tval, (i32 -1), (i32 imm:$cc), NZCV),
1279 (CSINVWr GPR32:$tval, WZR, (i32 imm:$cc))>;
1280 def : Pat<(AArch64csel GPR64:$tval, (i64 -1), (i32 imm:$cc), NZCV),
1281 (CSINVXr GPR64:$tval, XZR, (i32 imm:$cc))>;
1282 def : Pat<(AArch64csel (i32 -1), GPR32:$fval, (i32 imm:$cc), NZCV),
1283 (CSINVWr GPR32:$fval, WZR, (i32 (inv_cond_XFORM imm:$cc)))>;
1284 def : Pat<(AArch64csel (i64 -1), GPR64:$fval, (i32 imm:$cc), NZCV),
1285 (CSINVXr GPR64:$fval, XZR, (i32 (inv_cond_XFORM imm:$cc)))>;
1287 // The inverse of the condition code from the alias instruction is what is used
1288 // in the aliased instruction. The parser all ready inverts the condition code
1289 // for these aliases.
1290 def : InstAlias<"cset $dst, $cc",
1291 (CSINCWr GPR32:$dst, WZR, WZR, inv_ccode:$cc)>;
1292 def : InstAlias<"cset $dst, $cc",
1293 (CSINCXr GPR64:$dst, XZR, XZR, inv_ccode:$cc)>;
1295 def : InstAlias<"csetm $dst, $cc",
1296 (CSINVWr GPR32:$dst, WZR, WZR, inv_ccode:$cc)>;
1297 def : InstAlias<"csetm $dst, $cc",
1298 (CSINVXr GPR64:$dst, XZR, XZR, inv_ccode:$cc)>;
1300 def : InstAlias<"cinc $dst, $src, $cc",
1301 (CSINCWr GPR32:$dst, GPR32:$src, GPR32:$src, inv_ccode:$cc)>;
1302 def : InstAlias<"cinc $dst, $src, $cc",
1303 (CSINCXr GPR64:$dst, GPR64:$src, GPR64:$src, inv_ccode:$cc)>;
1305 def : InstAlias<"cinv $dst, $src, $cc",
1306 (CSINVWr GPR32:$dst, GPR32:$src, GPR32:$src, inv_ccode:$cc)>;
1307 def : InstAlias<"cinv $dst, $src, $cc",
1308 (CSINVXr GPR64:$dst, GPR64:$src, GPR64:$src, inv_ccode:$cc)>;
1310 def : InstAlias<"cneg $dst, $src, $cc",
1311 (CSNEGWr GPR32:$dst, GPR32:$src, GPR32:$src, inv_ccode:$cc)>;
1312 def : InstAlias<"cneg $dst, $src, $cc",
1313 (CSNEGXr GPR64:$dst, GPR64:$src, GPR64:$src, inv_ccode:$cc)>;
1315 //===----------------------------------------------------------------------===//
1316 // PC-relative instructions.
1317 //===----------------------------------------------------------------------===//
1318 let isReMaterializable = 1 in {
1319 let hasSideEffects = 0, mayStore = 0, mayLoad = 0 in {
1320 def ADR : ADRI<0, "adr", adrlabel, []>;
1321 } // hasSideEffects = 0
1323 def ADRP : ADRI<1, "adrp", adrplabel,
1324 [(set GPR64:$Xd, (AArch64adrp tglobaladdr:$label))]>;
1325 } // isReMaterializable = 1
1327 // page address of a constant pool entry, block address
1328 def : Pat<(AArch64adrp tconstpool:$cp), (ADRP tconstpool:$cp)>;
1329 def : Pat<(AArch64adrp tblockaddress:$cp), (ADRP tblockaddress:$cp)>;
1331 //===----------------------------------------------------------------------===//
1332 // Unconditional branch (register) instructions.
1333 //===----------------------------------------------------------------------===//
1335 let isReturn = 1, isTerminator = 1, isBarrier = 1 in {
1336 def RET : BranchReg<0b0010, "ret", []>;
1337 def DRPS : SpecialReturn<0b0101, "drps">;
1338 def ERET : SpecialReturn<0b0100, "eret">;
1339 } // isReturn = 1, isTerminator = 1, isBarrier = 1
1341 // Default to the LR register.
1342 def : InstAlias<"ret", (RET LR)>;
1344 let isCall = 1, Defs = [LR], Uses = [SP] in {
1345 def BLR : BranchReg<0b0001, "blr", [(AArch64call GPR64:$Rn)]>;
1348 let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
1349 def BR : BranchReg<0b0000, "br", [(brind GPR64:$Rn)]>;
1350 } // isBranch, isTerminator, isBarrier, isIndirectBranch
1352 // Create a separate pseudo-instruction for codegen to use so that we don't
1353 // flag lr as used in every function. It'll be restored before the RET by the
1354 // epilogue if it's legitimately used.
1355 def RET_ReallyLR : Pseudo<(outs), (ins), [(AArch64retflag)]>,
1356 Sched<[WriteBrReg]> {
1357 let isTerminator = 1;
1362 // This is a directive-like pseudo-instruction. The purpose is to insert an
1363 // R_AARCH64_TLSDESC_CALL relocation at the offset of the following instruction
1364 // (which in the usual case is a BLR).
1365 let hasSideEffects = 1 in
1366 def TLSDESCCALL : Pseudo<(outs), (ins i64imm:$sym), []>, Sched<[]> {
1367 let AsmString = ".tlsdesccall $sym";
1370 // FIXME: maybe the scratch register used shouldn't be fixed to X1?
1371 // FIXME: can "hasSideEffects be dropped?
1372 let isCall = 1, Defs = [LR, X0, X1], hasSideEffects = 1,
1373 isCodeGenOnly = 1 in
1375 : Pseudo<(outs), (ins i64imm:$sym),
1376 [(AArch64tlsdesc_callseq tglobaltlsaddr:$sym)]>,
1377 Sched<[WriteI, WriteLD, WriteI, WriteBrReg]>;
1378 def : Pat<(AArch64tlsdesc_callseq texternalsym:$sym),
1379 (TLSDESC_CALLSEQ texternalsym:$sym)>;
1381 //===----------------------------------------------------------------------===//
1382 // Conditional branch (immediate) instruction.
1383 //===----------------------------------------------------------------------===//
1384 def Bcc : BranchCond;
1386 //===----------------------------------------------------------------------===//
1387 // Compare-and-branch instructions.
1388 //===----------------------------------------------------------------------===//
1389 defm CBZ : CmpBranch<0, "cbz", AArch64cbz>;
1390 defm CBNZ : CmpBranch<1, "cbnz", AArch64cbnz>;
1392 //===----------------------------------------------------------------------===//
1393 // Test-bit-and-branch instructions.
1394 //===----------------------------------------------------------------------===//
1395 defm TBZ : TestBranch<0, "tbz", AArch64tbz>;
1396 defm TBNZ : TestBranch<1, "tbnz", AArch64tbnz>;
1398 //===----------------------------------------------------------------------===//
1399 // Unconditional branch (immediate) instructions.
1400 //===----------------------------------------------------------------------===//
1401 let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
1402 def B : BranchImm<0, "b", [(br bb:$addr)]>;
1403 } // isBranch, isTerminator, isBarrier
1405 let isCall = 1, Defs = [LR], Uses = [SP] in {
1406 def BL : CallImm<1, "bl", [(AArch64call tglobaladdr:$addr)]>;
1408 def : Pat<(AArch64call texternalsym:$func), (BL texternalsym:$func)>;
1410 //===----------------------------------------------------------------------===//
1411 // Exception generation instructions.
1412 //===----------------------------------------------------------------------===//
1413 def BRK : ExceptionGeneration<0b001, 0b00, "brk">;
1414 def DCPS1 : ExceptionGeneration<0b101, 0b01, "dcps1">;
1415 def DCPS2 : ExceptionGeneration<0b101, 0b10, "dcps2">;
1416 def DCPS3 : ExceptionGeneration<0b101, 0b11, "dcps3">;
1417 def HLT : ExceptionGeneration<0b010, 0b00, "hlt">;
1418 def HVC : ExceptionGeneration<0b000, 0b10, "hvc">;
1419 def SMC : ExceptionGeneration<0b000, 0b11, "smc">;
1420 def SVC : ExceptionGeneration<0b000, 0b01, "svc">;
1422 // DCPSn defaults to an immediate operand of zero if unspecified.
1423 def : InstAlias<"dcps1", (DCPS1 0)>;
1424 def : InstAlias<"dcps2", (DCPS2 0)>;
1425 def : InstAlias<"dcps3", (DCPS3 0)>;
1427 //===----------------------------------------------------------------------===//
1428 // Load instructions.
1429 //===----------------------------------------------------------------------===//
1431 // Pair (indexed, offset)
1432 defm LDPW : LoadPairOffset<0b00, 0, GPR32, simm7s4, "ldp">;
1433 defm LDPX : LoadPairOffset<0b10, 0, GPR64, simm7s8, "ldp">;
1434 defm LDPS : LoadPairOffset<0b00, 1, FPR32, simm7s4, "ldp">;
1435 defm LDPD : LoadPairOffset<0b01, 1, FPR64, simm7s8, "ldp">;
1436 defm LDPQ : LoadPairOffset<0b10, 1, FPR128, simm7s16, "ldp">;
1438 defm LDPSW : LoadPairOffset<0b01, 0, GPR64, simm7s4, "ldpsw">;
1440 // Pair (pre-indexed)
1441 def LDPWpre : LoadPairPreIdx<0b00, 0, GPR32, simm7s4, "ldp">;
1442 def LDPXpre : LoadPairPreIdx<0b10, 0, GPR64, simm7s8, "ldp">;
1443 def LDPSpre : LoadPairPreIdx<0b00, 1, FPR32, simm7s4, "ldp">;
1444 def LDPDpre : LoadPairPreIdx<0b01, 1, FPR64, simm7s8, "ldp">;
1445 def LDPQpre : LoadPairPreIdx<0b10, 1, FPR128, simm7s16, "ldp">;
1447 def LDPSWpre : LoadPairPreIdx<0b01, 0, GPR64, simm7s4, "ldpsw">;
1449 // Pair (post-indexed)
1450 def LDPWpost : LoadPairPostIdx<0b00, 0, GPR32, simm7s4, "ldp">;
1451 def LDPXpost : LoadPairPostIdx<0b10, 0, GPR64, simm7s8, "ldp">;
1452 def LDPSpost : LoadPairPostIdx<0b00, 1, FPR32, simm7s4, "ldp">;
1453 def LDPDpost : LoadPairPostIdx<0b01, 1, FPR64, simm7s8, "ldp">;
1454 def LDPQpost : LoadPairPostIdx<0b10, 1, FPR128, simm7s16, "ldp">;
1456 def LDPSWpost : LoadPairPostIdx<0b01, 0, GPR64, simm7s4, "ldpsw">;
1459 // Pair (no allocate)
1460 defm LDNPW : LoadPairNoAlloc<0b00, 0, GPR32, simm7s4, "ldnp">;
1461 defm LDNPX : LoadPairNoAlloc<0b10, 0, GPR64, simm7s8, "ldnp">;
1462 defm LDNPS : LoadPairNoAlloc<0b00, 1, FPR32, simm7s4, "ldnp">;
1463 defm LDNPD : LoadPairNoAlloc<0b01, 1, FPR64, simm7s8, "ldnp">;
1464 defm LDNPQ : LoadPairNoAlloc<0b10, 1, FPR128, simm7s16, "ldnp">;
1467 // (register offset)
1471 defm LDRBB : Load8RO<0b00, 0, 0b01, GPR32, "ldrb", i32, zextloadi8>;
1472 defm LDRHH : Load16RO<0b01, 0, 0b01, GPR32, "ldrh", i32, zextloadi16>;
1473 defm LDRW : Load32RO<0b10, 0, 0b01, GPR32, "ldr", i32, load>;
1474 defm LDRX : Load64RO<0b11, 0, 0b01, GPR64, "ldr", i64, load>;
1477 defm LDRB : Load8RO<0b00, 1, 0b01, FPR8, "ldr", untyped, load>;
1478 defm LDRH : Load16RO<0b01, 1, 0b01, FPR16, "ldr", f16, load>;
1479 defm LDRS : Load32RO<0b10, 1, 0b01, FPR32, "ldr", f32, load>;
1480 defm LDRD : Load64RO<0b11, 1, 0b01, FPR64, "ldr", f64, load>;
1481 defm LDRQ : Load128RO<0b00, 1, 0b11, FPR128, "ldr", f128, load>;
1483 // Load sign-extended half-word
1484 defm LDRSHW : Load16RO<0b01, 0, 0b11, GPR32, "ldrsh", i32, sextloadi16>;
1485 defm LDRSHX : Load16RO<0b01, 0, 0b10, GPR64, "ldrsh", i64, sextloadi16>;
1487 // Load sign-extended byte
1488 defm LDRSBW : Load8RO<0b00, 0, 0b11, GPR32, "ldrsb", i32, sextloadi8>;
1489 defm LDRSBX : Load8RO<0b00, 0, 0b10, GPR64, "ldrsb", i64, sextloadi8>;
1491 // Load sign-extended word
1492 defm LDRSW : Load32RO<0b10, 0, 0b10, GPR64, "ldrsw", i64, sextloadi32>;
1495 defm PRFM : PrefetchRO<0b11, 0, 0b10, "prfm">;
1497 // For regular load, we do not have any alignment requirement.
1498 // Thus, it is safe to directly map the vector loads with interesting
1499 // addressing modes.
1500 // FIXME: We could do the same for bitconvert to floating point vectors.
1501 multiclass ScalToVecROLoadPat<ROAddrMode ro, SDPatternOperator loadop,
1502 ValueType ScalTy, ValueType VecTy,
1503 Instruction LOADW, Instruction LOADX,
1505 def : Pat<(VecTy (scalar_to_vector (ScalTy
1506 (loadop (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$offset))))),
1507 (INSERT_SUBREG (VecTy (IMPLICIT_DEF)),
1508 (LOADW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$offset),
1511 def : Pat<(VecTy (scalar_to_vector (ScalTy
1512 (loadop (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$offset))))),
1513 (INSERT_SUBREG (VecTy (IMPLICIT_DEF)),
1514 (LOADX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$offset),
1518 let AddedComplexity = 10 in {
1519 defm : ScalToVecROLoadPat<ro8, extloadi8, i32, v8i8, LDRBroW, LDRBroX, bsub>;
1520 defm : ScalToVecROLoadPat<ro8, extloadi8, i32, v16i8, LDRBroW, LDRBroX, bsub>;
1522 defm : ScalToVecROLoadPat<ro16, extloadi16, i32, v4i16, LDRHroW, LDRHroX, hsub>;
1523 defm : ScalToVecROLoadPat<ro16, extloadi16, i32, v8i16, LDRHroW, LDRHroX, hsub>;
1525 defm : ScalToVecROLoadPat<ro16, load, i32, v4f16, LDRHroW, LDRHroX, hsub>;
1526 defm : ScalToVecROLoadPat<ro16, load, i32, v8f16, LDRHroW, LDRHroX, hsub>;
1528 defm : ScalToVecROLoadPat<ro32, load, i32, v2i32, LDRSroW, LDRSroX, ssub>;
1529 defm : ScalToVecROLoadPat<ro32, load, i32, v4i32, LDRSroW, LDRSroX, ssub>;
1531 defm : ScalToVecROLoadPat<ro32, load, f32, v2f32, LDRSroW, LDRSroX, ssub>;
1532 defm : ScalToVecROLoadPat<ro32, load, f32, v4f32, LDRSroW, LDRSroX, ssub>;
1534 defm : ScalToVecROLoadPat<ro64, load, i64, v2i64, LDRDroW, LDRDroX, dsub>;
1536 defm : ScalToVecROLoadPat<ro64, load, f64, v2f64, LDRDroW, LDRDroX, dsub>;
1539 def : Pat <(v1i64 (scalar_to_vector (i64
1540 (load (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
1541 ro_Wextend64:$extend))))),
1542 (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
1544 def : Pat <(v1i64 (scalar_to_vector (i64
1545 (load (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
1546 ro_Xextend64:$extend))))),
1547 (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
1550 // Match all load 64 bits width whose type is compatible with FPR64
1551 multiclass VecROLoadPat<ROAddrMode ro, ValueType VecTy,
1552 Instruction LOADW, Instruction LOADX> {
1554 def : Pat<(VecTy (load (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend))),
1555 (LOADW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
1557 def : Pat<(VecTy (load (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend))),
1558 (LOADX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
1561 let AddedComplexity = 10 in {
1562 let Predicates = [IsLE] in {
1563 // We must do vector loads with LD1 in big-endian.
1564 defm : VecROLoadPat<ro64, v2i32, LDRDroW, LDRDroX>;
1565 defm : VecROLoadPat<ro64, v2f32, LDRDroW, LDRDroX>;
1566 defm : VecROLoadPat<ro64, v8i8, LDRDroW, LDRDroX>;
1567 defm : VecROLoadPat<ro64, v4i16, LDRDroW, LDRDroX>;
1568 defm : VecROLoadPat<ro64, v4f16, LDRDroW, LDRDroX>;
1571 defm : VecROLoadPat<ro64, v1i64, LDRDroW, LDRDroX>;
1572 defm : VecROLoadPat<ro64, v1f64, LDRDroW, LDRDroX>;
1574 // Match all load 128 bits width whose type is compatible with FPR128
1575 let Predicates = [IsLE] in {
1576 // We must do vector loads with LD1 in big-endian.
1577 defm : VecROLoadPat<ro128, v2i64, LDRQroW, LDRQroX>;
1578 defm : VecROLoadPat<ro128, v2f64, LDRQroW, LDRQroX>;
1579 defm : VecROLoadPat<ro128, v4i32, LDRQroW, LDRQroX>;
1580 defm : VecROLoadPat<ro128, v4f32, LDRQroW, LDRQroX>;
1581 defm : VecROLoadPat<ro128, v8i16, LDRQroW, LDRQroX>;
1582 defm : VecROLoadPat<ro128, v8f16, LDRQroW, LDRQroX>;
1583 defm : VecROLoadPat<ro128, v16i8, LDRQroW, LDRQroX>;
1585 } // AddedComplexity = 10
1588 multiclass ExtLoadTo64ROPat<ROAddrMode ro, SDPatternOperator loadop,
1589 Instruction INSTW, Instruction INSTX> {
1590 def : Pat<(i64 (loadop (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend))),
1591 (SUBREG_TO_REG (i64 0),
1592 (INSTW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend),
1595 def : Pat<(i64 (loadop (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend))),
1596 (SUBREG_TO_REG (i64 0),
1597 (INSTX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend),
1601 let AddedComplexity = 10 in {
1602 defm : ExtLoadTo64ROPat<ro8, zextloadi8, LDRBBroW, LDRBBroX>;
1603 defm : ExtLoadTo64ROPat<ro16, zextloadi16, LDRHHroW, LDRHHroX>;
1604 defm : ExtLoadTo64ROPat<ro32, zextloadi32, LDRWroW, LDRWroX>;
1606 // zextloadi1 -> zextloadi8
1607 defm : ExtLoadTo64ROPat<ro8, zextloadi1, LDRBBroW, LDRBBroX>;
1609 // extload -> zextload
1610 defm : ExtLoadTo64ROPat<ro8, extloadi8, LDRBBroW, LDRBBroX>;
1611 defm : ExtLoadTo64ROPat<ro16, extloadi16, LDRHHroW, LDRHHroX>;
1612 defm : ExtLoadTo64ROPat<ro32, extloadi32, LDRWroW, LDRWroX>;
1614 // extloadi1 -> zextloadi8
1615 defm : ExtLoadTo64ROPat<ro8, extloadi1, LDRBBroW, LDRBBroX>;
1620 multiclass ExtLoadTo32ROPat<ROAddrMode ro, SDPatternOperator loadop,
1621 Instruction INSTW, Instruction INSTX> {
1622 def : Pat<(i32 (loadop (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend))),
1623 (INSTW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
1625 def : Pat<(i32 (loadop (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend))),
1626 (INSTX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
1630 let AddedComplexity = 10 in {
1631 // extload -> zextload
1632 defm : ExtLoadTo32ROPat<ro8, extloadi8, LDRBBroW, LDRBBroX>;
1633 defm : ExtLoadTo32ROPat<ro16, extloadi16, LDRHHroW, LDRHHroX>;
1634 defm : ExtLoadTo32ROPat<ro32, extloadi32, LDRWroW, LDRWroX>;
1636 // zextloadi1 -> zextloadi8
1637 defm : ExtLoadTo32ROPat<ro8, zextloadi1, LDRBBroW, LDRBBroX>;
1641 // (unsigned immediate)
1643 defm LDRX : LoadUI<0b11, 0, 0b01, GPR64, uimm12s8, "ldr",
1645 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)))]>;
1646 defm LDRW : LoadUI<0b10, 0, 0b01, GPR32, uimm12s4, "ldr",
1648 (load (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)))]>;
1649 defm LDRB : LoadUI<0b00, 1, 0b01, FPR8, uimm12s1, "ldr",
1651 (load (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset)))]>;
1652 defm LDRH : LoadUI<0b01, 1, 0b01, FPR16, uimm12s2, "ldr",
1653 [(set (f16 FPR16:$Rt),
1654 (load (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset)))]>;
1655 defm LDRS : LoadUI<0b10, 1, 0b01, FPR32, uimm12s4, "ldr",
1656 [(set (f32 FPR32:$Rt),
1657 (load (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)))]>;
1658 defm LDRD : LoadUI<0b11, 1, 0b01, FPR64, uimm12s8, "ldr",
1659 [(set (f64 FPR64:$Rt),
1660 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)))]>;
1661 defm LDRQ : LoadUI<0b00, 1, 0b11, FPR128, uimm12s16, "ldr",
1662 [(set (f128 FPR128:$Rt),
1663 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)))]>;
1665 // For regular load, we do not have any alignment requirement.
1666 // Thus, it is safe to directly map the vector loads with interesting
1667 // addressing modes.
1668 // FIXME: We could do the same for bitconvert to floating point vectors.
1669 def : Pat <(v8i8 (scalar_to_vector (i32
1670 (extloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))))),
1671 (INSERT_SUBREG (v8i8 (IMPLICIT_DEF)),
1672 (LDRBui GPR64sp:$Rn, uimm12s1:$offset), bsub)>;
1673 def : Pat <(v16i8 (scalar_to_vector (i32
1674 (extloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))))),
1675 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
1676 (LDRBui GPR64sp:$Rn, uimm12s1:$offset), bsub)>;
1677 def : Pat <(v4i16 (scalar_to_vector (i32
1678 (extloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))))),
1679 (INSERT_SUBREG (v4i16 (IMPLICIT_DEF)),
1680 (LDRHui GPR64sp:$Rn, uimm12s2:$offset), hsub)>;
1681 def : Pat <(v8i16 (scalar_to_vector (i32
1682 (extloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))))),
1683 (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)),
1684 (LDRHui GPR64sp:$Rn, uimm12s2:$offset), hsub)>;
1685 def : Pat <(v2i32 (scalar_to_vector (i32
1686 (load (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))))),
1687 (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)),
1688 (LDRSui GPR64sp:$Rn, uimm12s4:$offset), ssub)>;
1689 def : Pat <(v4i32 (scalar_to_vector (i32
1690 (load (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))))),
1691 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
1692 (LDRSui GPR64sp:$Rn, uimm12s4:$offset), ssub)>;
1693 def : Pat <(v1i64 (scalar_to_vector (i64
1694 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))))),
1695 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1696 def : Pat <(v2i64 (scalar_to_vector (i64
1697 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))))),
1698 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
1699 (LDRDui GPR64sp:$Rn, uimm12s8:$offset), dsub)>;
1701 // Match all load 64 bits width whose type is compatible with FPR64
1702 let Predicates = [IsLE] in {
1703 // We must use LD1 to perform vector loads in big-endian.
1704 def : Pat<(v2f32 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1705 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1706 def : Pat<(v8i8 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1707 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1708 def : Pat<(v4i16 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1709 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1710 def : Pat<(v2i32 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1711 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1712 def : Pat<(v4f16 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1713 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1715 def : Pat<(v1f64 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1716 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1717 def : Pat<(v1i64 (load (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))),
1718 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;
1720 // Match all load 128 bits width whose type is compatible with FPR128
1721 let Predicates = [IsLE] in {
1722 // We must use LD1 to perform vector loads in big-endian.
1723 def : Pat<(v4f32 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1724 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1725 def : Pat<(v2f64 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1726 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1727 def : Pat<(v16i8 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1728 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1729 def : Pat<(v8i16 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1730 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1731 def : Pat<(v4i32 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1732 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1733 def : Pat<(v2i64 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1734 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1735 def : Pat<(v8f16 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1736 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1738 def : Pat<(f128 (load (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset))),
1739 (LDRQui GPR64sp:$Rn, uimm12s16:$offset)>;
1741 defm LDRHH : LoadUI<0b01, 0, 0b01, GPR32, uimm12s2, "ldrh",
1743 (zextloadi16 (am_indexed16 GPR64sp:$Rn,
1744 uimm12s2:$offset)))]>;
1745 defm LDRBB : LoadUI<0b00, 0, 0b01, GPR32, uimm12s1, "ldrb",
1747 (zextloadi8 (am_indexed8 GPR64sp:$Rn,
1748 uimm12s1:$offset)))]>;
1750 def : Pat<(i64 (zextloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1751 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
1752 def : Pat<(i64 (zextloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))),
1753 (SUBREG_TO_REG (i64 0), (LDRHHui GPR64sp:$Rn, uimm12s2:$offset), sub_32)>;
1755 // zextloadi1 -> zextloadi8
1756 def : Pat<(i32 (zextloadi1 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1757 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
1758 def : Pat<(i64 (zextloadi1 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1759 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
1761 // extload -> zextload
1762 def : Pat<(i32 (extloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))),
1763 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
1764 def : Pat<(i32 (extloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1765 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
1766 def : Pat<(i32 (extloadi1 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1767 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
1768 def : Pat<(i64 (extloadi32 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))),
1769 (SUBREG_TO_REG (i64 0), (LDRWui GPR64sp:$Rn, uimm12s4:$offset), sub_32)>;
1770 def : Pat<(i64 (extloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))),
1771 (SUBREG_TO_REG (i64 0), (LDRHHui GPR64sp:$Rn, uimm12s2:$offset), sub_32)>;
1772 def : Pat<(i64 (extloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1773 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
1774 def : Pat<(i64 (extloadi1 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))),
1775 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
1777 // load sign-extended half-word
1778 defm LDRSHW : LoadUI<0b01, 0, 0b11, GPR32, uimm12s2, "ldrsh",
1780 (sextloadi16 (am_indexed16 GPR64sp:$Rn,
1781 uimm12s2:$offset)))]>;
1782 defm LDRSHX : LoadUI<0b01, 0, 0b10, GPR64, uimm12s2, "ldrsh",
1784 (sextloadi16 (am_indexed16 GPR64sp:$Rn,
1785 uimm12s2:$offset)))]>;
1787 // load sign-extended byte
1788 defm LDRSBW : LoadUI<0b00, 0, 0b11, GPR32, uimm12s1, "ldrsb",
1790 (sextloadi8 (am_indexed8 GPR64sp:$Rn,
1791 uimm12s1:$offset)))]>;
1792 defm LDRSBX : LoadUI<0b00, 0, 0b10, GPR64, uimm12s1, "ldrsb",
1794 (sextloadi8 (am_indexed8 GPR64sp:$Rn,
1795 uimm12s1:$offset)))]>;
1797 // load sign-extended word
1798 defm LDRSW : LoadUI<0b10, 0, 0b10, GPR64, uimm12s4, "ldrsw",
1800 (sextloadi32 (am_indexed32 GPR64sp:$Rn,
1801 uimm12s4:$offset)))]>;
1803 // load zero-extended word
1804 def : Pat<(i64 (zextloadi32 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))),
1805 (SUBREG_TO_REG (i64 0), (LDRWui GPR64sp:$Rn, uimm12s4:$offset), sub_32)>;
1808 def PRFMui : PrefetchUI<0b11, 0, 0b10, "prfm",
1809 [(AArch64Prefetch imm:$Rt,
1810 (am_indexed64 GPR64sp:$Rn,
1811 uimm12s8:$offset))]>;
1813 def : InstAlias<"prfm $Rt, [$Rn]", (PRFMui prfop:$Rt, GPR64sp:$Rn, 0)>;
1817 def LDRWl : LoadLiteral<0b00, 0, GPR32, "ldr">;
1818 def LDRXl : LoadLiteral<0b01, 0, GPR64, "ldr">;
1819 def LDRSl : LoadLiteral<0b00, 1, FPR32, "ldr">;
1820 def LDRDl : LoadLiteral<0b01, 1, FPR64, "ldr">;
1821 def LDRQl : LoadLiteral<0b10, 1, FPR128, "ldr">;
1823 // load sign-extended word
1824 def LDRSWl : LoadLiteral<0b10, 0, GPR64, "ldrsw">;
1827 def PRFMl : PrefetchLiteral<0b11, 0, "prfm", []>;
1828 // [(AArch64Prefetch imm:$Rt, tglobaladdr:$label)]>;
1831 // (unscaled immediate)
1832 defm LDURX : LoadUnscaled<0b11, 0, 0b01, GPR64, "ldur",
1834 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset)))]>;
1835 defm LDURW : LoadUnscaled<0b10, 0, 0b01, GPR32, "ldur",
1837 (load (am_unscaled32 GPR64sp:$Rn, simm9:$offset)))]>;
1838 defm LDURB : LoadUnscaled<0b00, 1, 0b01, FPR8, "ldur",
1840 (load (am_unscaled8 GPR64sp:$Rn, simm9:$offset)))]>;
1841 defm LDURH : LoadUnscaled<0b01, 1, 0b01, FPR16, "ldur",
1843 (load (am_unscaled16 GPR64sp:$Rn, simm9:$offset)))]>;
1844 defm LDURS : LoadUnscaled<0b10, 1, 0b01, FPR32, "ldur",
1845 [(set (f32 FPR32:$Rt),
1846 (load (am_unscaled32 GPR64sp:$Rn, simm9:$offset)))]>;
1847 defm LDURD : LoadUnscaled<0b11, 1, 0b01, FPR64, "ldur",
1848 [(set (f64 FPR64:$Rt),
1849 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset)))]>;
1850 defm LDURQ : LoadUnscaled<0b00, 1, 0b11, FPR128, "ldur",
1851 [(set (f128 FPR128:$Rt),
1852 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset)))]>;
1855 : LoadUnscaled<0b01, 0, 0b01, GPR32, "ldurh",
1857 (zextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)))]>;
1859 : LoadUnscaled<0b00, 0, 0b01, GPR32, "ldurb",
1861 (zextloadi8 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)))]>;
1863 // Match all load 64 bits width whose type is compatible with FPR64
1864 let Predicates = [IsLE] in {
1865 def : Pat<(v2f32 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1866 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1867 def : Pat<(v2i32 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1868 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1869 def : Pat<(v4i16 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1870 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1871 def : Pat<(v8i8 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1872 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1873 def : Pat<(v4f16 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1874 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1876 def : Pat<(v1f64 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1877 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1878 def : Pat<(v1i64 (load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
1879 (LDURDi GPR64sp:$Rn, simm9:$offset)>;
1881 // Match all load 128 bits width whose type is compatible with FPR128
1882 let Predicates = [IsLE] in {
1883 def : Pat<(v2f64 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1884 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1885 def : Pat<(v2i64 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1886 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1887 def : Pat<(v4f32 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1888 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1889 def : Pat<(v4i32 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1890 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1891 def : Pat<(v8i16 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1892 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1893 def : Pat<(v16i8 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1894 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1895 def : Pat<(v8f16 (load (am_unscaled128 GPR64sp:$Rn, simm9:$offset))),
1896 (LDURQi GPR64sp:$Rn, simm9:$offset)>;
1900 def : Pat<(i32 (extloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
1901 (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
1902 def : Pat<(i32 (extloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1903 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
1904 def : Pat<(i32 (extloadi1 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1905 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
1906 def : Pat<(i64 (extloadi32 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))),
1907 (SUBREG_TO_REG (i64 0), (LDURWi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1908 def : Pat<(i64 (extloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
1909 (SUBREG_TO_REG (i64 0), (LDURHHi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1910 def : Pat<(i64 (extloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1911 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1912 def : Pat<(i64 (extloadi1 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1913 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1915 def : Pat<(i32 (zextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
1916 (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
1917 def : Pat<(i32 (zextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1918 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
1919 def : Pat<(i32 (zextloadi1 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1920 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
1921 def : Pat<(i64 (zextloadi32 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))),
1922 (SUBREG_TO_REG (i64 0), (LDURWi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1923 def : Pat<(i64 (zextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
1924 (SUBREG_TO_REG (i64 0), (LDURHHi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1925 def : Pat<(i64 (zextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1926 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1927 def : Pat<(i64 (zextloadi1 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1928 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1932 // LDR mnemonics fall back to LDUR for negative or unaligned offsets.
1934 // Define new assembler match classes as we want to only match these when
1935 // the don't otherwise match the scaled addressing mode for LDR/STR. Don't
1936 // associate a DiagnosticType either, as we want the diagnostic for the
1937 // canonical form (the scaled operand) to take precedence.
1938 class SImm9OffsetOperand<int Width> : AsmOperandClass {
1939 let Name = "SImm9OffsetFB" # Width;
1940 let PredicateMethod = "isSImm9OffsetFB<" # Width # ">";
1941 let RenderMethod = "addImmOperands";
1944 def SImm9OffsetFB8Operand : SImm9OffsetOperand<8>;
1945 def SImm9OffsetFB16Operand : SImm9OffsetOperand<16>;
1946 def SImm9OffsetFB32Operand : SImm9OffsetOperand<32>;
1947 def SImm9OffsetFB64Operand : SImm9OffsetOperand<64>;
1948 def SImm9OffsetFB128Operand : SImm9OffsetOperand<128>;
1950 def simm9_offset_fb8 : Operand<i64> {
1951 let ParserMatchClass = SImm9OffsetFB8Operand;
1953 def simm9_offset_fb16 : Operand<i64> {
1954 let ParserMatchClass = SImm9OffsetFB16Operand;
1956 def simm9_offset_fb32 : Operand<i64> {
1957 let ParserMatchClass = SImm9OffsetFB32Operand;
1959 def simm9_offset_fb64 : Operand<i64> {
1960 let ParserMatchClass = SImm9OffsetFB64Operand;
1962 def simm9_offset_fb128 : Operand<i64> {
1963 let ParserMatchClass = SImm9OffsetFB128Operand;
1966 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1967 (LDURXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
1968 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1969 (LDURWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
1970 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1971 (LDURBi FPR8:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
1972 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1973 (LDURHi FPR16:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
1974 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1975 (LDURSi FPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
1976 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1977 (LDURDi FPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
1978 def : InstAlias<"ldr $Rt, [$Rn, $offset]",
1979 (LDURQi FPR128:$Rt, GPR64sp:$Rn, simm9_offset_fb128:$offset), 0>;
1982 def : Pat<(i64 (zextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))),
1983 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1984 def : Pat<(i64 (zextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
1985 (SUBREG_TO_REG (i64 0), (LDURHHi GPR64sp:$Rn, simm9:$offset), sub_32)>;
1987 // load sign-extended half-word
1989 : LoadUnscaled<0b01, 0, 0b11, GPR32, "ldursh",
1991 (sextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)))]>;
1993 : LoadUnscaled<0b01, 0, 0b10, GPR64, "ldursh",
1995 (sextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)))]>;
1997 // load sign-extended byte
1999 : LoadUnscaled<0b00, 0, 0b11, GPR32, "ldursb",
2001 (sextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)))]>;
2003 : LoadUnscaled<0b00, 0, 0b10, GPR64, "ldursb",
2005 (sextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)))]>;
2007 // load sign-extended word
2009 : LoadUnscaled<0b10, 0, 0b10, GPR64, "ldursw",
2011 (sextloadi32 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)))]>;
2013 // zero and sign extending aliases from generic LDR* mnemonics to LDUR*.
2014 def : InstAlias<"ldrb $Rt, [$Rn, $offset]",
2015 (LDURBBi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
2016 def : InstAlias<"ldrh $Rt, [$Rn, $offset]",
2017 (LDURHHi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
2018 def : InstAlias<"ldrsb $Rt, [$Rn, $offset]",
2019 (LDURSBWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
2020 def : InstAlias<"ldrsb $Rt, [$Rn, $offset]",
2021 (LDURSBXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
2022 def : InstAlias<"ldrsh $Rt, [$Rn, $offset]",
2023 (LDURSHWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
2024 def : InstAlias<"ldrsh $Rt, [$Rn, $offset]",
2025 (LDURSHXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
2026 def : InstAlias<"ldrsw $Rt, [$Rn, $offset]",
2027 (LDURSWi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
2030 defm PRFUM : PrefetchUnscaled<0b11, 0, 0b10, "prfum",
2031 [(AArch64Prefetch imm:$Rt,
2032 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))]>;
2035 // (unscaled immediate, unprivileged)
2036 defm LDTRX : LoadUnprivileged<0b11, 0, 0b01, GPR64, "ldtr">;
2037 defm LDTRW : LoadUnprivileged<0b10, 0, 0b01, GPR32, "ldtr">;
2039 defm LDTRH : LoadUnprivileged<0b01, 0, 0b01, GPR32, "ldtrh">;
2040 defm LDTRB : LoadUnprivileged<0b00, 0, 0b01, GPR32, "ldtrb">;
2042 // load sign-extended half-word
2043 defm LDTRSHW : LoadUnprivileged<0b01, 0, 0b11, GPR32, "ldtrsh">;
2044 defm LDTRSHX : LoadUnprivileged<0b01, 0, 0b10, GPR64, "ldtrsh">;
2046 // load sign-extended byte
2047 defm LDTRSBW : LoadUnprivileged<0b00, 0, 0b11, GPR32, "ldtrsb">;
2048 defm LDTRSBX : LoadUnprivileged<0b00, 0, 0b10, GPR64, "ldtrsb">;
2050 // load sign-extended word
2051 defm LDTRSW : LoadUnprivileged<0b10, 0, 0b10, GPR64, "ldtrsw">;
2054 // (immediate pre-indexed)
2055 def LDRWpre : LoadPreIdx<0b10, 0, 0b01, GPR32, "ldr">;
2056 def LDRXpre : LoadPreIdx<0b11, 0, 0b01, GPR64, "ldr">;
2057 def LDRBpre : LoadPreIdx<0b00, 1, 0b01, FPR8, "ldr">;
2058 def LDRHpre : LoadPreIdx<0b01, 1, 0b01, FPR16, "ldr">;
2059 def LDRSpre : LoadPreIdx<0b10, 1, 0b01, FPR32, "ldr">;
2060 def LDRDpre : LoadPreIdx<0b11, 1, 0b01, FPR64, "ldr">;
2061 def LDRQpre : LoadPreIdx<0b00, 1, 0b11, FPR128, "ldr">;
2063 // load sign-extended half-word
2064 def LDRSHWpre : LoadPreIdx<0b01, 0, 0b11, GPR32, "ldrsh">;
2065 def LDRSHXpre : LoadPreIdx<0b01, 0, 0b10, GPR64, "ldrsh">;
2067 // load sign-extended byte
2068 def LDRSBWpre : LoadPreIdx<0b00, 0, 0b11, GPR32, "ldrsb">;
2069 def LDRSBXpre : LoadPreIdx<0b00, 0, 0b10, GPR64, "ldrsb">;
2071 // load zero-extended byte
2072 def LDRBBpre : LoadPreIdx<0b00, 0, 0b01, GPR32, "ldrb">;
2073 def LDRHHpre : LoadPreIdx<0b01, 0, 0b01, GPR32, "ldrh">;
2075 // load sign-extended word
2076 def LDRSWpre : LoadPreIdx<0b10, 0, 0b10, GPR64, "ldrsw">;
2079 // (immediate post-indexed)
2080 def LDRWpost : LoadPostIdx<0b10, 0, 0b01, GPR32, "ldr">;
2081 def LDRXpost : LoadPostIdx<0b11, 0, 0b01, GPR64, "ldr">;
2082 def LDRBpost : LoadPostIdx<0b00, 1, 0b01, FPR8, "ldr">;
2083 def LDRHpost : LoadPostIdx<0b01, 1, 0b01, FPR16, "ldr">;
2084 def LDRSpost : LoadPostIdx<0b10, 1, 0b01, FPR32, "ldr">;
2085 def LDRDpost : LoadPostIdx<0b11, 1, 0b01, FPR64, "ldr">;
2086 def LDRQpost : LoadPostIdx<0b00, 1, 0b11, FPR128, "ldr">;
2088 // load sign-extended half-word
2089 def LDRSHWpost : LoadPostIdx<0b01, 0, 0b11, GPR32, "ldrsh">;
2090 def LDRSHXpost : LoadPostIdx<0b01, 0, 0b10, GPR64, "ldrsh">;
2092 // load sign-extended byte
2093 def LDRSBWpost : LoadPostIdx<0b00, 0, 0b11, GPR32, "ldrsb">;
2094 def LDRSBXpost : LoadPostIdx<0b00, 0, 0b10, GPR64, "ldrsb">;
2096 // load zero-extended byte
2097 def LDRBBpost : LoadPostIdx<0b00, 0, 0b01, GPR32, "ldrb">;
2098 def LDRHHpost : LoadPostIdx<0b01, 0, 0b01, GPR32, "ldrh">;
2100 // load sign-extended word
2101 def LDRSWpost : LoadPostIdx<0b10, 0, 0b10, GPR64, "ldrsw">;
2103 //===----------------------------------------------------------------------===//
2104 // Store instructions.
2105 //===----------------------------------------------------------------------===//
2107 // Pair (indexed, offset)
2108 // FIXME: Use dedicated range-checked addressing mode operand here.
2109 defm STPW : StorePairOffset<0b00, 0, GPR32, simm7s4, "stp">;
2110 defm STPX : StorePairOffset<0b10, 0, GPR64, simm7s8, "stp">;
2111 defm STPS : StorePairOffset<0b00, 1, FPR32, simm7s4, "stp">;
2112 defm STPD : StorePairOffset<0b01, 1, FPR64, simm7s8, "stp">;
2113 defm STPQ : StorePairOffset<0b10, 1, FPR128, simm7s16, "stp">;
2115 // Pair (pre-indexed)
2116 def STPWpre : StorePairPreIdx<0b00, 0, GPR32, simm7s4, "stp">;
2117 def STPXpre : StorePairPreIdx<0b10, 0, GPR64, simm7s8, "stp">;
2118 def STPSpre : StorePairPreIdx<0b00, 1, FPR32, simm7s4, "stp">;
2119 def STPDpre : StorePairPreIdx<0b01, 1, FPR64, simm7s8, "stp">;
2120 def STPQpre : StorePairPreIdx<0b10, 1, FPR128, simm7s16, "stp">;
2122 // Pair (pre-indexed)
2123 def STPWpost : StorePairPostIdx<0b00, 0, GPR32, simm7s4, "stp">;
2124 def STPXpost : StorePairPostIdx<0b10, 0, GPR64, simm7s8, "stp">;
2125 def STPSpost : StorePairPostIdx<0b00, 1, FPR32, simm7s4, "stp">;
2126 def STPDpost : StorePairPostIdx<0b01, 1, FPR64, simm7s8, "stp">;
2127 def STPQpost : StorePairPostIdx<0b10, 1, FPR128, simm7s16, "stp">;
2129 // Pair (no allocate)
2130 defm STNPW : StorePairNoAlloc<0b00, 0, GPR32, simm7s4, "stnp">;
2131 defm STNPX : StorePairNoAlloc<0b10, 0, GPR64, simm7s8, "stnp">;
2132 defm STNPS : StorePairNoAlloc<0b00, 1, FPR32, simm7s4, "stnp">;
2133 defm STNPD : StorePairNoAlloc<0b01, 1, FPR64, simm7s8, "stnp">;
2134 defm STNPQ : StorePairNoAlloc<0b10, 1, FPR128, simm7s16, "stnp">;
2137 // (Register offset)
2140 defm STRBB : Store8RO< 0b00, 0, 0b00, GPR32, "strb", i32, truncstorei8>;
2141 defm STRHH : Store16RO<0b01, 0, 0b00, GPR32, "strh", i32, truncstorei16>;
2142 defm STRW : Store32RO<0b10, 0, 0b00, GPR32, "str", i32, store>;
2143 defm STRX : Store64RO<0b11, 0, 0b00, GPR64, "str", i64, store>;
2147 defm STRB : Store8RO< 0b00, 1, 0b00, FPR8, "str", untyped, store>;
2148 defm STRH : Store16RO<0b01, 1, 0b00, FPR16, "str", f16, store>;
2149 defm STRS : Store32RO<0b10, 1, 0b00, FPR32, "str", f32, store>;
2150 defm STRD : Store64RO<0b11, 1, 0b00, FPR64, "str", f64, store>;
2151 defm STRQ : Store128RO<0b00, 1, 0b10, FPR128, "str", f128, store>;
2153 let Predicates = [UseSTRQro], AddedComplexity = 10 in {
2154 def : Pat<(store (f128 FPR128:$Rt),
2155 (ro_Windexed128 GPR64sp:$Rn, GPR32:$Rm,
2156 ro_Wextend128:$extend)),
2157 (STRQroW FPR128:$Rt, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend128:$extend)>;
2158 def : Pat<(store (f128 FPR128:$Rt),
2159 (ro_Xindexed128 GPR64sp:$Rn, GPR64:$Rm,
2160 ro_Xextend128:$extend)),
2161 (STRQroX FPR128:$Rt, GPR64sp:$Rn, GPR64:$Rm, ro_Wextend128:$extend)>;
2164 multiclass TruncStoreFrom64ROPat<ROAddrMode ro, SDPatternOperator storeop,
2165 Instruction STRW, Instruction STRX> {
2167 def : Pat<(storeop GPR64:$Rt,
2168 (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)),
2169 (STRW (EXTRACT_SUBREG GPR64:$Rt, sub_32),
2170 GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
2172 def : Pat<(storeop GPR64:$Rt,
2173 (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)),
2174 (STRX (EXTRACT_SUBREG GPR64:$Rt, sub_32),
2175 GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
2178 let AddedComplexity = 10 in {
2180 defm : TruncStoreFrom64ROPat<ro8, truncstorei8, STRBBroW, STRBBroX>;
2181 defm : TruncStoreFrom64ROPat<ro16, truncstorei16, STRHHroW, STRHHroX>;
2182 defm : TruncStoreFrom64ROPat<ro32, truncstorei32, STRWroW, STRWroX>;
2185 multiclass VecROStorePat<ROAddrMode ro, ValueType VecTy, RegisterClass FPR,
2186 Instruction STRW, Instruction STRX> {
2187 def : Pat<(store (VecTy FPR:$Rt),
2188 (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)),
2189 (STRW FPR:$Rt, GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
2191 def : Pat<(store (VecTy FPR:$Rt),
2192 (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)),
2193 (STRX FPR:$Rt, GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
2196 let AddedComplexity = 10 in {
2197 // Match all store 64 bits width whose type is compatible with FPR64
2198 let Predicates = [IsLE] in {
2199 // We must use ST1 to store vectors in big-endian.
2200 defm : VecROStorePat<ro64, v2i32, FPR64, STRDroW, STRDroX>;
2201 defm : VecROStorePat<ro64, v2f32, FPR64, STRDroW, STRDroX>;
2202 defm : VecROStorePat<ro64, v4i16, FPR64, STRDroW, STRDroX>;
2203 defm : VecROStorePat<ro64, v8i8, FPR64, STRDroW, STRDroX>;
2204 defm : VecROStorePat<ro64, v4f16, FPR64, STRDroW, STRDroX>;
2207 defm : VecROStorePat<ro64, v1i64, FPR64, STRDroW, STRDroX>;
2208 defm : VecROStorePat<ro64, v1f64, FPR64, STRDroW, STRDroX>;
2210 // Match all store 128 bits width whose type is compatible with FPR128
2211 let Predicates = [IsLE, UseSTRQro] in {
2212 // We must use ST1 to store vectors in big-endian.
2213 defm : VecROStorePat<ro128, v2i64, FPR128, STRQroW, STRQroX>;
2214 defm : VecROStorePat<ro128, v2f64, FPR128, STRQroW, STRQroX>;
2215 defm : VecROStorePat<ro128, v4i32, FPR128, STRQroW, STRQroX>;
2216 defm : VecROStorePat<ro128, v4f32, FPR128, STRQroW, STRQroX>;
2217 defm : VecROStorePat<ro128, v8i16, FPR128, STRQroW, STRQroX>;
2218 defm : VecROStorePat<ro128, v16i8, FPR128, STRQroW, STRQroX>;
2219 defm : VecROStorePat<ro128, v8f16, FPR128, STRQroW, STRQroX>;
2221 } // AddedComplexity = 10
2223 // Match stores from lane 0 to the appropriate subreg's store.
2224 multiclass VecROStoreLane0Pat<ROAddrMode ro, SDPatternOperator storeop,
2225 ValueType VecTy, ValueType STy,
2226 SubRegIndex SubRegIdx,
2227 Instruction STRW, Instruction STRX> {
2229 def : Pat<(storeop (STy (vector_extract (VecTy VecListOne128:$Vt), 0)),
2230 (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)),
2231 (STRW (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx),
2232 GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
2234 def : Pat<(storeop (STy (vector_extract (VecTy VecListOne128:$Vt), 0)),
2235 (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)),
2236 (STRX (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx),
2237 GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
2240 let AddedComplexity = 19 in {
2241 defm : VecROStoreLane0Pat<ro16, truncstorei16, v8i16, i32, hsub, STRHroW, STRHroX>;
2242 defm : VecROStoreLane0Pat<ro16, store , v8i16, i16, hsub, STRHroW, STRHroX>;
2243 defm : VecROStoreLane0Pat<ro32, truncstorei32, v4i32, i32, ssub, STRSroW, STRSroX>;
2244 defm : VecROStoreLane0Pat<ro32, store , v4i32, i32, ssub, STRSroW, STRSroX>;
2245 defm : VecROStoreLane0Pat<ro32, store , v4f32, f32, ssub, STRSroW, STRSroX>;
2246 defm : VecROStoreLane0Pat<ro64, store , v2i64, i64, dsub, STRDroW, STRDroX>;
2247 defm : VecROStoreLane0Pat<ro64, store , v2f64, f64, dsub, STRDroW, STRDroX>;
2251 // (unsigned immediate)
2252 defm STRX : StoreUIz<0b11, 0, 0b00, GPR64z, uimm12s8, "str",
2254 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))]>;
2255 defm STRW : StoreUIz<0b10, 0, 0b00, GPR32z, uimm12s4, "str",
2257 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))]>;
2258 defm STRB : StoreUI<0b00, 1, 0b00, FPR8, uimm12s1, "str",
2260 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))]>;
2261 defm STRH : StoreUI<0b01, 1, 0b00, FPR16, uimm12s2, "str",
2262 [(store (f16 FPR16:$Rt),
2263 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))]>;
2264 defm STRS : StoreUI<0b10, 1, 0b00, FPR32, uimm12s4, "str",
2265 [(store (f32 FPR32:$Rt),
2266 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))]>;
2267 defm STRD : StoreUI<0b11, 1, 0b00, FPR64, uimm12s8, "str",
2268 [(store (f64 FPR64:$Rt),
2269 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))]>;
2270 defm STRQ : StoreUI<0b00, 1, 0b10, FPR128, uimm12s16, "str", []>;
2272 defm STRHH : StoreUIz<0b01, 0, 0b00, GPR32z, uimm12s2, "strh",
2273 [(truncstorei16 GPR32z:$Rt,
2274 (am_indexed16 GPR64sp:$Rn,
2275 uimm12s2:$offset))]>;
2276 defm STRBB : StoreUIz<0b00, 0, 0b00, GPR32z, uimm12s1, "strb",
2277 [(truncstorei8 GPR32z:$Rt,
2278 (am_indexed8 GPR64sp:$Rn,
2279 uimm12s1:$offset))]>;
2281 // Match all store 64 bits width whose type is compatible with FPR64
2282 let AddedComplexity = 10 in {
2283 let Predicates = [IsLE] in {
2284 // We must use ST1 to store vectors in big-endian.
2285 def : Pat<(store (v2f32 FPR64:$Rt),
2286 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2287 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2288 def : Pat<(store (v8i8 FPR64:$Rt),
2289 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2290 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2291 def : Pat<(store (v4i16 FPR64:$Rt),
2292 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2293 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2294 def : Pat<(store (v2i32 FPR64:$Rt),
2295 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2296 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2297 def : Pat<(store (v4f16 FPR64:$Rt),
2298 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2299 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2301 def : Pat<(store (v1f64 FPR64:$Rt),
2302 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2303 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2304 def : Pat<(store (v1i64 FPR64:$Rt),
2305 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),
2306 (STRDui FPR64:$Rt, GPR64sp:$Rn, uimm12s8:$offset)>;
2308 // Match all store 128 bits width whose type is compatible with FPR128
2309 let Predicates = [IsLE] in {
2310 // We must use ST1 to store vectors in big-endian.
2311 def : Pat<(store (v4f32 FPR128:$Rt),
2312 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2313 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2314 def : Pat<(store (v2f64 FPR128:$Rt),
2315 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2316 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2317 def : Pat<(store (v16i8 FPR128:$Rt),
2318 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2319 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2320 def : Pat<(store (v8i16 FPR128:$Rt),
2321 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2322 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2323 def : Pat<(store (v4i32 FPR128:$Rt),
2324 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2325 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2326 def : Pat<(store (v2i64 FPR128:$Rt),
2327 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2328 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2329 def : Pat<(store (v8f16 FPR128:$Rt),
2330 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2331 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2333 def : Pat<(store (f128 FPR128:$Rt),
2334 (am_indexed128 GPR64sp:$Rn, uimm12s16:$offset)),
2335 (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>;
2338 def : Pat<(truncstorei32 GPR64:$Rt,
2339 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)),
2340 (STRWui (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, uimm12s4:$offset)>;
2341 def : Pat<(truncstorei16 GPR64:$Rt,
2342 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset)),
2343 (STRHHui (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, uimm12s2:$offset)>;
2344 def : Pat<(truncstorei8 GPR64:$Rt, (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset)),
2345 (STRBBui (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, uimm12s1:$offset)>;
2347 } // AddedComplexity = 10
2350 // (unscaled immediate)
2351 defm STURX : StoreUnscaled<0b11, 0, 0b00, GPR64, "stur",
2353 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))]>;
2354 defm STURW : StoreUnscaled<0b10, 0, 0b00, GPR32, "stur",
2356 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))]>;
2357 defm STURB : StoreUnscaled<0b00, 1, 0b00, FPR8, "stur",
2359 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))]>;
2360 defm STURH : StoreUnscaled<0b01, 1, 0b00, FPR16, "stur",
2361 [(store (f16 FPR16:$Rt),
2362 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))]>;
2363 defm STURS : StoreUnscaled<0b10, 1, 0b00, FPR32, "stur",
2364 [(store (f32 FPR32:$Rt),
2365 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))]>;
2366 defm STURD : StoreUnscaled<0b11, 1, 0b00, FPR64, "stur",
2367 [(store (f64 FPR64:$Rt),
2368 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))]>;
2369 defm STURQ : StoreUnscaled<0b00, 1, 0b10, FPR128, "stur",
2370 [(store (f128 FPR128:$Rt),
2371 (am_unscaled128 GPR64sp:$Rn, simm9:$offset))]>;
2372 defm STURHH : StoreUnscaled<0b01, 0, 0b00, GPR32, "sturh",
2373 [(truncstorei16 GPR32:$Rt,
2374 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))]>;
2375 defm STURBB : StoreUnscaled<0b00, 0, 0b00, GPR32, "sturb",
2376 [(truncstorei8 GPR32:$Rt,
2377 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))]>;
2379 // Match all store 64 bits width whose type is compatible with FPR64
2380 let Predicates = [IsLE] in {
2381 // We must use ST1 to store vectors in big-endian.
2382 def : Pat<(store (v2f32 FPR64:$Rt),
2383 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2384 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2385 def : Pat<(store (v8i8 FPR64:$Rt),
2386 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2387 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2388 def : Pat<(store (v4i16 FPR64:$Rt),
2389 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2390 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2391 def : Pat<(store (v2i32 FPR64:$Rt),
2392 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2393 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2394 def : Pat<(store (v4f16 FPR64:$Rt),
2395 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2396 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2398 def : Pat<(store (v1f64 FPR64:$Rt), (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2399 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2400 def : Pat<(store (v1i64 FPR64:$Rt), (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
2401 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2403 // Match all store 128 bits width whose type is compatible with FPR128
2404 let Predicates = [IsLE] in {
2405 // We must use ST1 to store vectors in big-endian.
2406 def : Pat<(store (v4f32 FPR128:$Rt),
2407 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2408 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2409 def : Pat<(store (v2f64 FPR128:$Rt),
2410 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2411 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2412 def : Pat<(store (v16i8 FPR128:$Rt),
2413 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2414 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2415 def : Pat<(store (v8i16 FPR128:$Rt),
2416 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2417 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2418 def : Pat<(store (v4i32 FPR128:$Rt),
2419 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2420 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2421 def : Pat<(store (v2i64 FPR128:$Rt),
2422 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2423 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2424 def : Pat<(store (v2f64 FPR128:$Rt),
2425 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2426 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2427 def : Pat<(store (v8f16 FPR128:$Rt),
2428 (am_unscaled128 GPR64sp:$Rn, simm9:$offset)),
2429 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9:$offset)>;
2432 // unscaled i64 truncating stores
2433 def : Pat<(truncstorei32 GPR64:$Rt, (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
2434 (STURWi (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, simm9:$offset)>;
2435 def : Pat<(truncstorei16 GPR64:$Rt, (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
2436 (STURHHi (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, simm9:$offset)>;
2437 def : Pat<(truncstorei8 GPR64:$Rt, (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
2438 (STURBBi (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, simm9:$offset)>;
2441 // STR mnemonics fall back to STUR for negative or unaligned offsets.
2442 def : InstAlias<"str $Rt, [$Rn, $offset]",
2443 (STURXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
2444 def : InstAlias<"str $Rt, [$Rn, $offset]",
2445 (STURWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
2446 def : InstAlias<"str $Rt, [$Rn, $offset]",
2447 (STURBi FPR8:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
2448 def : InstAlias<"str $Rt, [$Rn, $offset]",
2449 (STURHi FPR16:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
2450 def : InstAlias<"str $Rt, [$Rn, $offset]",
2451 (STURSi FPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
2452 def : InstAlias<"str $Rt, [$Rn, $offset]",
2453 (STURDi FPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
2454 def : InstAlias<"str $Rt, [$Rn, $offset]",
2455 (STURQi FPR128:$Rt, GPR64sp:$Rn, simm9_offset_fb128:$offset), 0>;
2457 def : InstAlias<"strb $Rt, [$Rn, $offset]",
2458 (STURBBi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
2459 def : InstAlias<"strh $Rt, [$Rn, $offset]",
2460 (STURHHi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
2463 // (unscaled immediate, unprivileged)
2464 defm STTRW : StoreUnprivileged<0b10, 0, 0b00, GPR32, "sttr">;
2465 defm STTRX : StoreUnprivileged<0b11, 0, 0b00, GPR64, "sttr">;
2467 defm STTRH : StoreUnprivileged<0b01, 0, 0b00, GPR32, "sttrh">;
2468 defm STTRB : StoreUnprivileged<0b00, 0, 0b00, GPR32, "sttrb">;
2471 // (immediate pre-indexed)
2472 def STRWpre : StorePreIdx<0b10, 0, 0b00, GPR32, "str", pre_store, i32>;
2473 def STRXpre : StorePreIdx<0b11, 0, 0b00, GPR64, "str", pre_store, i64>;
2474 def STRBpre : StorePreIdx<0b00, 1, 0b00, FPR8, "str", pre_store, untyped>;
2475 def STRHpre : StorePreIdx<0b01, 1, 0b00, FPR16, "str", pre_store, f16>;
2476 def STRSpre : StorePreIdx<0b10, 1, 0b00, FPR32, "str", pre_store, f32>;
2477 def STRDpre : StorePreIdx<0b11, 1, 0b00, FPR64, "str", pre_store, f64>;
2478 def STRQpre : StorePreIdx<0b00, 1, 0b10, FPR128, "str", pre_store, f128>;
2480 def STRBBpre : StorePreIdx<0b00, 0, 0b00, GPR32, "strb", pre_truncsti8, i32>;
2481 def STRHHpre : StorePreIdx<0b01, 0, 0b00, GPR32, "strh", pre_truncsti16, i32>;
2484 def : Pat<(pre_truncsti32 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
2485 (STRWpre (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$addr,
2487 def : Pat<(pre_truncsti16 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
2488 (STRHHpre (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$addr,
2490 def : Pat<(pre_truncsti8 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
2491 (STRBBpre (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$addr,
2494 def : Pat<(pre_store (v8i8 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2495 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2496 def : Pat<(pre_store (v4i16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2497 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2498 def : Pat<(pre_store (v2i32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2499 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2500 def : Pat<(pre_store (v2f32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2501 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2502 def : Pat<(pre_store (v1i64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2503 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2504 def : Pat<(pre_store (v1f64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2505 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2506 def : Pat<(pre_store (v4f16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2507 (STRDpre FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2509 def : Pat<(pre_store (v16i8 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2510 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2511 def : Pat<(pre_store (v8i16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2512 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2513 def : Pat<(pre_store (v4i32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2514 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2515 def : Pat<(pre_store (v4f32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2516 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2517 def : Pat<(pre_store (v2i64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2518 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2519 def : Pat<(pre_store (v2f64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2520 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2521 def : Pat<(pre_store (v8f16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2522 (STRQpre FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2525 // (immediate post-indexed)
2526 def STRWpost : StorePostIdx<0b10, 0, 0b00, GPR32, "str", post_store, i32>;
2527 def STRXpost : StorePostIdx<0b11, 0, 0b00, GPR64, "str", post_store, i64>;
2528 def STRBpost : StorePostIdx<0b00, 1, 0b00, FPR8, "str", post_store, untyped>;
2529 def STRHpost : StorePostIdx<0b01, 1, 0b00, FPR16, "str", post_store, f16>;
2530 def STRSpost : StorePostIdx<0b10, 1, 0b00, FPR32, "str", post_store, f32>;
2531 def STRDpost : StorePostIdx<0b11, 1, 0b00, FPR64, "str", post_store, f64>;
2532 def STRQpost : StorePostIdx<0b00, 1, 0b10, FPR128, "str", post_store, f128>;
2534 def STRBBpost : StorePostIdx<0b00, 0, 0b00, GPR32, "strb", post_truncsti8, i32>;
2535 def STRHHpost : StorePostIdx<0b01, 0, 0b00, GPR32, "strh", post_truncsti16, i32>;
2538 def : Pat<(post_truncsti32 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
2539 (STRWpost (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$addr,
2541 def : Pat<(post_truncsti16 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
2542 (STRHHpost (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$addr,
2544 def : Pat<(post_truncsti8 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
2545 (STRBBpost (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$addr,
2548 def : Pat<(post_store (v8i8 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2549 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2550 def : Pat<(post_store (v4i16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2551 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2552 def : Pat<(post_store (v2i32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2553 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2554 def : Pat<(post_store (v2f32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2555 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2556 def : Pat<(post_store (v1i64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2557 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2558 def : Pat<(post_store (v1f64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2559 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2560 def : Pat<(post_store (v4f16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
2561 (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
2563 def : Pat<(post_store (v16i8 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2564 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2565 def : Pat<(post_store (v8i16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2566 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2567 def : Pat<(post_store (v4i32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2568 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2569 def : Pat<(post_store (v4f32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2570 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2571 def : Pat<(post_store (v2i64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2572 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2573 def : Pat<(post_store (v2f64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2574 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2575 def : Pat<(post_store (v8f16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
2576 (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
2578 //===----------------------------------------------------------------------===//
2579 // Load/store exclusive instructions.
2580 //===----------------------------------------------------------------------===//
2582 def LDARW : LoadAcquire <0b10, 1, 1, 0, 1, GPR32, "ldar">;
2583 def LDARX : LoadAcquire <0b11, 1, 1, 0, 1, GPR64, "ldar">;
2584 def LDARB : LoadAcquire <0b00, 1, 1, 0, 1, GPR32, "ldarb">;
2585 def LDARH : LoadAcquire <0b01, 1, 1, 0, 1, GPR32, "ldarh">;
2587 def LDAXRW : LoadExclusive <0b10, 0, 1, 0, 1, GPR32, "ldaxr">;
2588 def LDAXRX : LoadExclusive <0b11, 0, 1, 0, 1, GPR64, "ldaxr">;
2589 def LDAXRB : LoadExclusive <0b00, 0, 1, 0, 1, GPR32, "ldaxrb">;
2590 def LDAXRH : LoadExclusive <0b01, 0, 1, 0, 1, GPR32, "ldaxrh">;
2592 def LDXRW : LoadExclusive <0b10, 0, 1, 0, 0, GPR32, "ldxr">;
2593 def LDXRX : LoadExclusive <0b11, 0, 1, 0, 0, GPR64, "ldxr">;
2594 def LDXRB : LoadExclusive <0b00, 0, 1, 0, 0, GPR32, "ldxrb">;
2595 def LDXRH : LoadExclusive <0b01, 0, 1, 0, 0, GPR32, "ldxrh">;
2597 def STLRW : StoreRelease <0b10, 1, 0, 0, 1, GPR32, "stlr">;
2598 def STLRX : StoreRelease <0b11, 1, 0, 0, 1, GPR64, "stlr">;
2599 def STLRB : StoreRelease <0b00, 1, 0, 0, 1, GPR32, "stlrb">;
2600 def STLRH : StoreRelease <0b01, 1, 0, 0, 1, GPR32, "stlrh">;
2602 def STLXRW : StoreExclusive<0b10, 0, 0, 0, 1, GPR32, "stlxr">;
2603 def STLXRX : StoreExclusive<0b11, 0, 0, 0, 1, GPR64, "stlxr">;
2604 def STLXRB : StoreExclusive<0b00, 0, 0, 0, 1, GPR32, "stlxrb">;
2605 def STLXRH : StoreExclusive<0b01, 0, 0, 0, 1, GPR32, "stlxrh">;
2607 def STXRW : StoreExclusive<0b10, 0, 0, 0, 0, GPR32, "stxr">;
2608 def STXRX : StoreExclusive<0b11, 0, 0, 0, 0, GPR64, "stxr">;
2609 def STXRB : StoreExclusive<0b00, 0, 0, 0, 0, GPR32, "stxrb">;
2610 def STXRH : StoreExclusive<0b01, 0, 0, 0, 0, GPR32, "stxrh">;
2612 def LDAXPW : LoadExclusivePair<0b10, 0, 1, 1, 1, GPR32, "ldaxp">;
2613 def LDAXPX : LoadExclusivePair<0b11, 0, 1, 1, 1, GPR64, "ldaxp">;
2615 def LDXPW : LoadExclusivePair<0b10, 0, 1, 1, 0, GPR32, "ldxp">;
2616 def LDXPX : LoadExclusivePair<0b11, 0, 1, 1, 0, GPR64, "ldxp">;
2618 def STLXPW : StoreExclusivePair<0b10, 0, 0, 1, 1, GPR32, "stlxp">;
2619 def STLXPX : StoreExclusivePair<0b11, 0, 0, 1, 1, GPR64, "stlxp">;
2621 def STXPW : StoreExclusivePair<0b10, 0, 0, 1, 0, GPR32, "stxp">;
2622 def STXPX : StoreExclusivePair<0b11, 0, 0, 1, 0, GPR64, "stxp">;
2624 let Predicates = [HasV8_1a] in {
2625 // v8.1a "Limited Order Region" extension load-acquire instructions
2626 def LDLARW : LoadAcquire <0b10, 1, 1, 0, 0, GPR32, "ldlar">;
2627 def LDLARX : LoadAcquire <0b11, 1, 1, 0, 0, GPR64, "ldlar">;
2628 def LDLARB : LoadAcquire <0b00, 1, 1, 0, 0, GPR32, "ldlarb">;
2629 def LDLARH : LoadAcquire <0b01, 1, 1, 0, 0, GPR32, "ldlarh">;
2631 // v8.1a "Limited Order Region" extension store-release instructions
2632 def STLLRW : StoreRelease <0b10, 1, 0, 0, 0, GPR32, "stllr">;
2633 def STLLRX : StoreRelease <0b11, 1, 0, 0, 0, GPR64, "stllr">;
2634 def STLLRB : StoreRelease <0b00, 1, 0, 0, 0, GPR32, "stllrb">;
2635 def STLLRH : StoreRelease <0b01, 1, 0, 0, 0, GPR32, "stllrh">;
2638 //===----------------------------------------------------------------------===//
2639 // Scaled floating point to integer conversion instructions.
2640 //===----------------------------------------------------------------------===//
2642 defm FCVTAS : FPToIntegerUnscaled<0b00, 0b100, "fcvtas", int_aarch64_neon_fcvtas>;
2643 defm FCVTAU : FPToIntegerUnscaled<0b00, 0b101, "fcvtau", int_aarch64_neon_fcvtau>;
2644 defm FCVTMS : FPToIntegerUnscaled<0b10, 0b000, "fcvtms", int_aarch64_neon_fcvtms>;
2645 defm FCVTMU : FPToIntegerUnscaled<0b10, 0b001, "fcvtmu", int_aarch64_neon_fcvtmu>;
2646 defm FCVTNS : FPToIntegerUnscaled<0b00, 0b000, "fcvtns", int_aarch64_neon_fcvtns>;
2647 defm FCVTNU : FPToIntegerUnscaled<0b00, 0b001, "fcvtnu", int_aarch64_neon_fcvtnu>;
2648 defm FCVTPS : FPToIntegerUnscaled<0b01, 0b000, "fcvtps", int_aarch64_neon_fcvtps>;
2649 defm FCVTPU : FPToIntegerUnscaled<0b01, 0b001, "fcvtpu", int_aarch64_neon_fcvtpu>;
2650 defm FCVTZS : FPToIntegerUnscaled<0b11, 0b000, "fcvtzs", fp_to_sint>;
2651 defm FCVTZU : FPToIntegerUnscaled<0b11, 0b001, "fcvtzu", fp_to_uint>;
2652 defm FCVTZS : FPToIntegerScaled<0b11, 0b000, "fcvtzs", fp_to_sint>;
2653 defm FCVTZU : FPToIntegerScaled<0b11, 0b001, "fcvtzu", fp_to_uint>;
2655 multiclass FPToIntegerIntPats<Intrinsic round, string INST> {
2656 def : Pat<(i32 (round f16:$Rn)), (!cast<Instruction>(INST # UWHr) $Rn)>;
2657 def : Pat<(i64 (round f16:$Rn)), (!cast<Instruction>(INST # UXHr) $Rn)>;
2658 def : Pat<(i32 (round f32:$Rn)), (!cast<Instruction>(INST # UWSr) $Rn)>;
2659 def : Pat<(i64 (round f32:$Rn)), (!cast<Instruction>(INST # UXSr) $Rn)>;
2660 def : Pat<(i32 (round f64:$Rn)), (!cast<Instruction>(INST # UWDr) $Rn)>;
2661 def : Pat<(i64 (round f64:$Rn)), (!cast<Instruction>(INST # UXDr) $Rn)>;
2663 def : Pat<(i32 (round (fmul f16:$Rn, fixedpoint_f16_i32:$scale))),
2664 (!cast<Instruction>(INST # SWHri) $Rn, $scale)>;
2665 def : Pat<(i64 (round (fmul f16:$Rn, fixedpoint_f16_i64:$scale))),
2666 (!cast<Instruction>(INST # SXHri) $Rn, $scale)>;
2667 def : Pat<(i32 (round (fmul f32:$Rn, fixedpoint_f32_i32:$scale))),
2668 (!cast<Instruction>(INST # SWSri) $Rn, $scale)>;
2669 def : Pat<(i64 (round (fmul f32:$Rn, fixedpoint_f32_i64:$scale))),
2670 (!cast<Instruction>(INST # SXSri) $Rn, $scale)>;
2671 def : Pat<(i32 (round (fmul f64:$Rn, fixedpoint_f64_i32:$scale))),
2672 (!cast<Instruction>(INST # SWDri) $Rn, $scale)>;
2673 def : Pat<(i64 (round (fmul f64:$Rn, fixedpoint_f64_i64:$scale))),
2674 (!cast<Instruction>(INST # SXDri) $Rn, $scale)>;
2677 defm : FPToIntegerIntPats<int_aarch64_neon_fcvtzs, "FCVTZS">;
2678 defm : FPToIntegerIntPats<int_aarch64_neon_fcvtzu, "FCVTZU">;
2680 multiclass FPToIntegerPats<SDNode to_int, SDNode round, string INST> {
2681 def : Pat<(i32 (to_int (round f32:$Rn))),
2682 (!cast<Instruction>(INST # UWSr) f32:$Rn)>;
2683 def : Pat<(i64 (to_int (round f32:$Rn))),
2684 (!cast<Instruction>(INST # UXSr) f32:$Rn)>;
2685 def : Pat<(i32 (to_int (round f64:$Rn))),
2686 (!cast<Instruction>(INST # UWDr) f64:$Rn)>;
2687 def : Pat<(i64 (to_int (round f64:$Rn))),
2688 (!cast<Instruction>(INST # UXDr) f64:$Rn)>;
2691 defm : FPToIntegerPats<fp_to_sint, fceil, "FCVTPS">;
2692 defm : FPToIntegerPats<fp_to_uint, fceil, "FCVTPU">;
2693 defm : FPToIntegerPats<fp_to_sint, ffloor, "FCVTMS">;
2694 defm : FPToIntegerPats<fp_to_uint, ffloor, "FCVTMU">;
2695 defm : FPToIntegerPats<fp_to_sint, ftrunc, "FCVTZS">;
2696 defm : FPToIntegerPats<fp_to_uint, ftrunc, "FCVTZU">;
2697 defm : FPToIntegerPats<fp_to_sint, fround, "FCVTAS">;
2698 defm : FPToIntegerPats<fp_to_uint, fround, "FCVTAU">;
2700 //===----------------------------------------------------------------------===//
2701 // Scaled integer to floating point conversion instructions.
2702 //===----------------------------------------------------------------------===//
2704 defm SCVTF : IntegerToFP<0, "scvtf", sint_to_fp>;
2705 defm UCVTF : IntegerToFP<1, "ucvtf", uint_to_fp>;
2707 //===----------------------------------------------------------------------===//
2708 // Unscaled integer to floating point conversion instruction.
2709 //===----------------------------------------------------------------------===//
2711 defm FMOV : UnscaledConversion<"fmov">;
2713 // Add pseudo ops for FMOV 0 so we can mark them as isReMaterializable
2714 let isReMaterializable = 1, isCodeGenOnly = 1, isAsCheapAsAMove = 1 in {
2715 def FMOVH0 : Pseudo<(outs FPR16:$Rd), (ins), [(set f16:$Rd, (fpimm0))]>,
2717 def FMOVS0 : Pseudo<(outs FPR32:$Rd), (ins), [(set f32:$Rd, (fpimm0))]>,
2719 def FMOVD0 : Pseudo<(outs FPR64:$Rd), (ins), [(set f64:$Rd, (fpimm0))]>,
2722 // Similarly add aliases
2723 def : InstAlias<"fmov $Rd, #0.0", (FMOVWHr FPR16:$Rd, WZR), 0>,
2724 Requires<[HasFullFP16]>;
2725 def : InstAlias<"fmov $Rd, #0.0", (FMOVWSr FPR32:$Rd, WZR), 0>;
2726 def : InstAlias<"fmov $Rd, #0.0", (FMOVXDr FPR64:$Rd, XZR), 0>;
2728 //===----------------------------------------------------------------------===//
2729 // Floating point conversion instruction.
2730 //===----------------------------------------------------------------------===//
2732 defm FCVT : FPConversion<"fcvt">;
2734 //===----------------------------------------------------------------------===//
2735 // Floating point single operand instructions.
2736 //===----------------------------------------------------------------------===//
2738 defm FABS : SingleOperandFPData<0b0001, "fabs", fabs>;
2739 defm FMOV : SingleOperandFPData<0b0000, "fmov">;
2740 defm FNEG : SingleOperandFPData<0b0010, "fneg", fneg>;
2741 defm FRINTA : SingleOperandFPData<0b1100, "frinta", fround>;
2742 defm FRINTI : SingleOperandFPData<0b1111, "frinti", fnearbyint>;
2743 defm FRINTM : SingleOperandFPData<0b1010, "frintm", ffloor>;
2744 defm FRINTN : SingleOperandFPData<0b1000, "frintn", int_aarch64_neon_frintn>;
2745 defm FRINTP : SingleOperandFPData<0b1001, "frintp", fceil>;
2747 def : Pat<(v1f64 (int_aarch64_neon_frintn (v1f64 FPR64:$Rn))),
2748 (FRINTNDr FPR64:$Rn)>;
2750 defm FRINTX : SingleOperandFPData<0b1110, "frintx", frint>;
2751 defm FRINTZ : SingleOperandFPData<0b1011, "frintz", ftrunc>;
2753 let SchedRW = [WriteFDiv] in {
2754 defm FSQRT : SingleOperandFPData<0b0011, "fsqrt", fsqrt>;
2757 //===----------------------------------------------------------------------===//
2758 // Floating point two operand instructions.
2759 //===----------------------------------------------------------------------===//
2761 defm FADD : TwoOperandFPData<0b0010, "fadd", fadd>;
2762 let SchedRW = [WriteFDiv] in {
2763 defm FDIV : TwoOperandFPData<0b0001, "fdiv", fdiv>;
2765 defm FMAXNM : TwoOperandFPData<0b0110, "fmaxnm", fmaxnum>;
2766 defm FMAX : TwoOperandFPData<0b0100, "fmax", fmaxnan>;
2767 defm FMINNM : TwoOperandFPData<0b0111, "fminnm", fminnum>;
2768 defm FMIN : TwoOperandFPData<0b0101, "fmin", fminnan>;
2769 let SchedRW = [WriteFMul] in {
2770 defm FMUL : TwoOperandFPData<0b0000, "fmul", fmul>;
2771 defm FNMUL : TwoOperandFPDataNeg<0b1000, "fnmul", fmul>;
2773 defm FSUB : TwoOperandFPData<0b0011, "fsub", fsub>;
2775 def : Pat<(v1f64 (fmaxnan (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
2776 (FMAXDrr FPR64:$Rn, FPR64:$Rm)>;
2777 def : Pat<(v1f64 (fminnan (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
2778 (FMINDrr FPR64:$Rn, FPR64:$Rm)>;
2779 def : Pat<(v1f64 (fmaxnum (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
2780 (FMAXNMDrr FPR64:$Rn, FPR64:$Rm)>;
2781 def : Pat<(v1f64 (fminnum (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
2782 (FMINNMDrr FPR64:$Rn, FPR64:$Rm)>;
2784 //===----------------------------------------------------------------------===//
2785 // Floating point three operand instructions.
2786 //===----------------------------------------------------------------------===//
2788 defm FMADD : ThreeOperandFPData<0, 0, "fmadd", fma>;
2789 defm FMSUB : ThreeOperandFPData<0, 1, "fmsub",
2790 TriOpFrag<(fma node:$LHS, (fneg node:$MHS), node:$RHS)> >;
2791 defm FNMADD : ThreeOperandFPData<1, 0, "fnmadd",
2792 TriOpFrag<(fneg (fma node:$LHS, node:$MHS, node:$RHS))> >;
2793 defm FNMSUB : ThreeOperandFPData<1, 1, "fnmsub",
2794 TriOpFrag<(fma node:$LHS, node:$MHS, (fneg node:$RHS))> >;
2796 // The following def pats catch the case where the LHS of an FMA is negated.
2797 // The TriOpFrag above catches the case where the middle operand is negated.
2799 // N.b. FMSUB etc have the accumulator at the *end* of (outs), unlike
2800 // the NEON variant.
2801 def : Pat<(f32 (fma (fneg FPR32:$Rn), FPR32:$Rm, FPR32:$Ra)),
2802 (FMSUBSrrr FPR32:$Rn, FPR32:$Rm, FPR32:$Ra)>;
2804 def : Pat<(f64 (fma (fneg FPR64:$Rn), FPR64:$Rm, FPR64:$Ra)),
2805 (FMSUBDrrr FPR64:$Rn, FPR64:$Rm, FPR64:$Ra)>;
2807 // We handled -(a + b*c) for FNMADD above, now it's time for "(-a) + (-b)*c" and
2809 def : Pat<(f32 (fma (fneg FPR32:$Rn), FPR32:$Rm, (fneg FPR32:$Ra))),
2810 (FNMADDSrrr FPR32:$Rn, FPR32:$Rm, FPR32:$Ra)>;
2812 def : Pat<(f64 (fma (fneg FPR64:$Rn), FPR64:$Rm, (fneg FPR64:$Ra))),
2813 (FNMADDDrrr FPR64:$Rn, FPR64:$Rm, FPR64:$Ra)>;
2815 def : Pat<(f32 (fma FPR32:$Rn, (fneg FPR32:$Rm), (fneg FPR32:$Ra))),
2816 (FNMADDSrrr FPR32:$Rn, FPR32:$Rm, FPR32:$Ra)>;
2818 def : Pat<(f64 (fma FPR64:$Rn, (fneg FPR64:$Rm), (fneg FPR64:$Ra))),
2819 (FNMADDDrrr FPR64:$Rn, FPR64:$Rm, FPR64:$Ra)>;
2821 //===----------------------------------------------------------------------===//
2822 // Floating point comparison instructions.
2823 //===----------------------------------------------------------------------===//
2825 defm FCMPE : FPComparison<1, "fcmpe">;
2826 defm FCMP : FPComparison<0, "fcmp", AArch64fcmp>;
2828 //===----------------------------------------------------------------------===//
2829 // Floating point conditional comparison instructions.
2830 //===----------------------------------------------------------------------===//
2832 defm FCCMPE : FPCondComparison<1, "fccmpe">;
2833 defm FCCMP : FPCondComparison<0, "fccmp", AArch64fccmp>;
2835 //===----------------------------------------------------------------------===//
2836 // Floating point conditional select instruction.
2837 //===----------------------------------------------------------------------===//
2839 defm FCSEL : FPCondSelect<"fcsel">;
2841 // CSEL instructions providing f128 types need to be handled by a
2842 // pseudo-instruction since the eventual code will need to introduce basic
2843 // blocks and control flow.
2844 def F128CSEL : Pseudo<(outs FPR128:$Rd),
2845 (ins FPR128:$Rn, FPR128:$Rm, ccode:$cond),
2846 [(set (f128 FPR128:$Rd),
2847 (AArch64csel FPR128:$Rn, FPR128:$Rm,
2848 (i32 imm:$cond), NZCV))]> {
2850 let usesCustomInserter = 1;
2851 let hasNoSchedulingInfo = 1;
2855 //===----------------------------------------------------------------------===//
2856 // Floating point immediate move.
2857 //===----------------------------------------------------------------------===//
2859 let isReMaterializable = 1 in {
2860 defm FMOV : FPMoveImmediate<"fmov">;
2863 //===----------------------------------------------------------------------===//
2864 // Advanced SIMD two vector instructions.
2865 //===----------------------------------------------------------------------===//
2867 defm UABDL : SIMDLongThreeVectorBHSabdl<1, 0b0111, "uabdl",
2868 int_aarch64_neon_uabd>;
2869 // Match UABDL in log2-shuffle patterns.
2870 def : Pat<(abs (v8i16 (sub (zext (v8i8 V64:$opA)),
2871 (zext (v8i8 V64:$opB))))),
2872 (UABDLv8i8_v8i16 V64:$opA, V64:$opB)>;
2873 def : Pat<(xor (v8i16 (AArch64vashr v8i16:$src, (i32 15))),
2874 (v8i16 (add (sub (zext (v8i8 V64:$opA)),
2875 (zext (v8i8 V64:$opB))),
2876 (AArch64vashr v8i16:$src, (i32 15))))),
2877 (UABDLv8i8_v8i16 V64:$opA, V64:$opB)>;
2878 def : Pat<(abs (v8i16 (sub (zext (extract_high_v16i8 V128:$opA)),
2879 (zext (extract_high_v16i8 V128:$opB))))),
2880 (UABDLv16i8_v8i16 V128:$opA, V128:$opB)>;
2881 def : Pat<(xor (v8i16 (AArch64vashr v8i16:$src, (i32 15))),
2882 (v8i16 (add (sub (zext (extract_high_v16i8 V128:$opA)),
2883 (zext (extract_high_v16i8 V128:$opB))),
2884 (AArch64vashr v8i16:$src, (i32 15))))),
2885 (UABDLv16i8_v8i16 V128:$opA, V128:$opB)>;
2886 def : Pat<(abs (v4i32 (sub (zext (v4i16 V64:$opA)),
2887 (zext (v4i16 V64:$opB))))),
2888 (UABDLv4i16_v4i32 V64:$opA, V64:$opB)>;
2889 def : Pat<(abs (v4i32 (sub (zext (extract_high_v8i16 V128:$opA)),
2890 (zext (extract_high_v8i16 V128:$opB))))),
2891 (UABDLv8i16_v4i32 V128:$opA, V128:$opB)>;
2892 def : Pat<(abs (v2i64 (sub (zext (v2i32 V64:$opA)),
2893 (zext (v2i32 V64:$opB))))),
2894 (UABDLv2i32_v2i64 V64:$opA, V64:$opB)>;
2895 def : Pat<(abs (v2i64 (sub (zext (extract_high_v4i32 V128:$opA)),
2896 (zext (extract_high_v4i32 V128:$opB))))),
2897 (UABDLv4i32_v2i64 V128:$opA, V128:$opB)>;
2899 defm ABS : SIMDTwoVectorBHSD<0, 0b01011, "abs", abs>;
2900 defm CLS : SIMDTwoVectorBHS<0, 0b00100, "cls", int_aarch64_neon_cls>;
2901 defm CLZ : SIMDTwoVectorBHS<1, 0b00100, "clz", ctlz>;
2902 defm CMEQ : SIMDCmpTwoVector<0, 0b01001, "cmeq", AArch64cmeqz>;
2903 defm CMGE : SIMDCmpTwoVector<1, 0b01000, "cmge", AArch64cmgez>;
2904 defm CMGT : SIMDCmpTwoVector<0, 0b01000, "cmgt", AArch64cmgtz>;
2905 defm CMLE : SIMDCmpTwoVector<1, 0b01001, "cmle", AArch64cmlez>;
2906 defm CMLT : SIMDCmpTwoVector<0, 0b01010, "cmlt", AArch64cmltz>;
2907 defm CNT : SIMDTwoVectorB<0, 0b00, 0b00101, "cnt", ctpop>;
2908 defm FABS : SIMDTwoVectorFP<0, 1, 0b01111, "fabs", fabs>;
2910 defm FCMEQ : SIMDFPCmpTwoVector<0, 1, 0b01101, "fcmeq", AArch64fcmeqz>;
2911 defm FCMGE : SIMDFPCmpTwoVector<1, 1, 0b01100, "fcmge", AArch64fcmgez>;
2912 defm FCMGT : SIMDFPCmpTwoVector<0, 1, 0b01100, "fcmgt", AArch64fcmgtz>;
2913 defm FCMLE : SIMDFPCmpTwoVector<1, 1, 0b01101, "fcmle", AArch64fcmlez>;
2914 defm FCMLT : SIMDFPCmpTwoVector<0, 1, 0b01110, "fcmlt", AArch64fcmltz>;
2915 defm FCVTAS : SIMDTwoVectorFPToInt<0,0,0b11100, "fcvtas",int_aarch64_neon_fcvtas>;
2916 defm FCVTAU : SIMDTwoVectorFPToInt<1,0,0b11100, "fcvtau",int_aarch64_neon_fcvtau>;
2917 defm FCVTL : SIMDFPWidenTwoVector<0, 0, 0b10111, "fcvtl">;
2918 def : Pat<(v4f32 (int_aarch64_neon_vcvthf2fp (v4i16 V64:$Rn))),
2919 (FCVTLv4i16 V64:$Rn)>;
2920 def : Pat<(v4f32 (int_aarch64_neon_vcvthf2fp (extract_subvector (v8i16 V128:$Rn),
2922 (FCVTLv8i16 V128:$Rn)>;
2923 def : Pat<(v2f64 (fpextend (v2f32 V64:$Rn))), (FCVTLv2i32 V64:$Rn)>;
2924 def : Pat<(v2f64 (fpextend (v2f32 (extract_subvector (v4f32 V128:$Rn),
2926 (FCVTLv4i32 V128:$Rn)>;
2928 def : Pat<(v4f32 (fpextend (v4f16 V64:$Rn))), (FCVTLv4i16 V64:$Rn)>;
2929 def : Pat<(v4f32 (fpextend (v4f16 (extract_subvector (v8f16 V128:$Rn),
2931 (FCVTLv8i16 V128:$Rn)>;
2933 defm FCVTMS : SIMDTwoVectorFPToInt<0,0,0b11011, "fcvtms",int_aarch64_neon_fcvtms>;
2934 defm FCVTMU : SIMDTwoVectorFPToInt<1,0,0b11011, "fcvtmu",int_aarch64_neon_fcvtmu>;
2935 defm FCVTNS : SIMDTwoVectorFPToInt<0,0,0b11010, "fcvtns",int_aarch64_neon_fcvtns>;
2936 defm FCVTNU : SIMDTwoVectorFPToInt<1,0,0b11010, "fcvtnu",int_aarch64_neon_fcvtnu>;
2937 defm FCVTN : SIMDFPNarrowTwoVector<0, 0, 0b10110, "fcvtn">;
2938 def : Pat<(v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn))),
2939 (FCVTNv4i16 V128:$Rn)>;
2940 def : Pat<(concat_vectors V64:$Rd,
2941 (v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn)))),
2942 (FCVTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>;
2943 def : Pat<(v2f32 (fpround (v2f64 V128:$Rn))), (FCVTNv2i32 V128:$Rn)>;
2944 def : Pat<(v4f16 (fpround (v4f32 V128:$Rn))), (FCVTNv4i16 V128:$Rn)>;
2945 def : Pat<(concat_vectors V64:$Rd, (v2f32 (fpround (v2f64 V128:$Rn)))),
2946 (FCVTNv4i32 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>;
2947 defm FCVTPS : SIMDTwoVectorFPToInt<0,1,0b11010, "fcvtps",int_aarch64_neon_fcvtps>;
2948 defm FCVTPU : SIMDTwoVectorFPToInt<1,1,0b11010, "fcvtpu",int_aarch64_neon_fcvtpu>;
2949 defm FCVTXN : SIMDFPInexactCvtTwoVector<1, 0, 0b10110, "fcvtxn",
2950 int_aarch64_neon_fcvtxn>;
2951 defm FCVTZS : SIMDTwoVectorFPToInt<0, 1, 0b11011, "fcvtzs", fp_to_sint>;
2952 defm FCVTZU : SIMDTwoVectorFPToInt<1, 1, 0b11011, "fcvtzu", fp_to_uint>;
2954 def : Pat<(v4i16 (int_aarch64_neon_fcvtzs v4f16:$Rn)), (FCVTZSv4f16 $Rn)>;
2955 def : Pat<(v8i16 (int_aarch64_neon_fcvtzs v8f16:$Rn)), (FCVTZSv8f16 $Rn)>;
2956 def : Pat<(v2i32 (int_aarch64_neon_fcvtzs v2f32:$Rn)), (FCVTZSv2f32 $Rn)>;
2957 def : Pat<(v4i32 (int_aarch64_neon_fcvtzs v4f32:$Rn)), (FCVTZSv4f32 $Rn)>;
2958 def : Pat<(v2i64 (int_aarch64_neon_fcvtzs v2f64:$Rn)), (FCVTZSv2f64 $Rn)>;
2960 def : Pat<(v4i16 (int_aarch64_neon_fcvtzu v4f16:$Rn)), (FCVTZUv4f16 $Rn)>;
2961 def : Pat<(v8i16 (int_aarch64_neon_fcvtzu v8f16:$Rn)), (FCVTZUv8f16 $Rn)>;
2962 def : Pat<(v2i32 (int_aarch64_neon_fcvtzu v2f32:$Rn)), (FCVTZUv2f32 $Rn)>;
2963 def : Pat<(v4i32 (int_aarch64_neon_fcvtzu v4f32:$Rn)), (FCVTZUv4f32 $Rn)>;
2964 def : Pat<(v2i64 (int_aarch64_neon_fcvtzu v2f64:$Rn)), (FCVTZUv2f64 $Rn)>;
2966 defm FNEG : SIMDTwoVectorFP<1, 1, 0b01111, "fneg", fneg>;
2967 defm FRECPE : SIMDTwoVectorFP<0, 1, 0b11101, "frecpe", int_aarch64_neon_frecpe>;
2968 defm FRINTA : SIMDTwoVectorFP<1, 0, 0b11000, "frinta", fround>;
2969 defm FRINTI : SIMDTwoVectorFP<1, 1, 0b11001, "frinti", fnearbyint>;
2970 defm FRINTM : SIMDTwoVectorFP<0, 0, 0b11001, "frintm", ffloor>;
2971 defm FRINTN : SIMDTwoVectorFP<0, 0, 0b11000, "frintn", int_aarch64_neon_frintn>;
2972 defm FRINTP : SIMDTwoVectorFP<0, 1, 0b11000, "frintp", fceil>;
2973 defm FRINTX : SIMDTwoVectorFP<1, 0, 0b11001, "frintx", frint>;
2974 defm FRINTZ : SIMDTwoVectorFP<0, 1, 0b11001, "frintz", ftrunc>;
2975 defm FRSQRTE: SIMDTwoVectorFP<1, 1, 0b11101, "frsqrte", int_aarch64_neon_frsqrte>;
2976 defm FSQRT : SIMDTwoVectorFP<1, 1, 0b11111, "fsqrt", fsqrt>;
2977 defm NEG : SIMDTwoVectorBHSD<1, 0b01011, "neg",
2978 UnOpFrag<(sub immAllZerosV, node:$LHS)> >;
2979 defm NOT : SIMDTwoVectorB<1, 0b00, 0b00101, "not", vnot>;
2980 // Aliases for MVN -> NOT.
2981 def : InstAlias<"mvn{ $Vd.8b, $Vn.8b|.8b $Vd, $Vn}",
2982 (NOTv8i8 V64:$Vd, V64:$Vn)>;
2983 def : InstAlias<"mvn{ $Vd.16b, $Vn.16b|.16b $Vd, $Vn}",
2984 (NOTv16i8 V128:$Vd, V128:$Vn)>;
2986 def : Pat<(AArch64neg (v8i8 V64:$Rn)), (NEGv8i8 V64:$Rn)>;
2987 def : Pat<(AArch64neg (v16i8 V128:$Rn)), (NEGv16i8 V128:$Rn)>;
2988 def : Pat<(AArch64neg (v4i16 V64:$Rn)), (NEGv4i16 V64:$Rn)>;
2989 def : Pat<(AArch64neg (v8i16 V128:$Rn)), (NEGv8i16 V128:$Rn)>;
2990 def : Pat<(AArch64neg (v2i32 V64:$Rn)), (NEGv2i32 V64:$Rn)>;
2991 def : Pat<(AArch64neg (v4i32 V128:$Rn)), (NEGv4i32 V128:$Rn)>;
2992 def : Pat<(AArch64neg (v2i64 V128:$Rn)), (NEGv2i64 V128:$Rn)>;
2994 def : Pat<(AArch64not (v8i8 V64:$Rn)), (NOTv8i8 V64:$Rn)>;
2995 def : Pat<(AArch64not (v16i8 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
2996 def : Pat<(AArch64not (v4i16 V64:$Rn)), (NOTv8i8 V64:$Rn)>;
2997 def : Pat<(AArch64not (v8i16 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
2998 def : Pat<(AArch64not (v2i32 V64:$Rn)), (NOTv8i8 V64:$Rn)>;
2999 def : Pat<(AArch64not (v1i64 V64:$Rn)), (NOTv8i8 V64:$Rn)>;
3000 def : Pat<(AArch64not (v4i32 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
3001 def : Pat<(AArch64not (v2i64 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
3003 def : Pat<(vnot (v4i16 V64:$Rn)), (NOTv8i8 V64:$Rn)>;
3004 def : Pat<(vnot (v8i16 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
3005 def : Pat<(vnot (v2i32 V64:$Rn)), (NOTv8i8 V64:$Rn)>;
3006 def : Pat<(vnot (v4i32 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
3007 def : Pat<(vnot (v2i64 V128:$Rn)), (NOTv16i8 V128:$Rn)>;
3009 defm RBIT : SIMDTwoVectorB<1, 0b01, 0b00101, "rbit", int_aarch64_neon_rbit>;
3010 defm REV16 : SIMDTwoVectorB<0, 0b00, 0b00001, "rev16", AArch64rev16>;
3011 defm REV32 : SIMDTwoVectorBH<1, 0b00000, "rev32", AArch64rev32>;
3012 defm REV64 : SIMDTwoVectorBHS<0, 0b00000, "rev64", AArch64rev64>;
3013 defm SADALP : SIMDLongTwoVectorTied<0, 0b00110, "sadalp",
3014 BinOpFrag<(add node:$LHS, (int_aarch64_neon_saddlp node:$RHS))> >;
3015 defm SADDLP : SIMDLongTwoVector<0, 0b00010, "saddlp", int_aarch64_neon_saddlp>;
3016 defm SCVTF : SIMDTwoVectorIntToFP<0, 0, 0b11101, "scvtf", sint_to_fp>;
3017 defm SHLL : SIMDVectorLShiftLongBySizeBHS;
3018 defm SQABS : SIMDTwoVectorBHSD<0, 0b00111, "sqabs", int_aarch64_neon_sqabs>;
3019 defm SQNEG : SIMDTwoVectorBHSD<1, 0b00111, "sqneg", int_aarch64_neon_sqneg>;
3020 defm SQXTN : SIMDMixedTwoVector<0, 0b10100, "sqxtn", int_aarch64_neon_sqxtn>;
3021 defm SQXTUN : SIMDMixedTwoVector<1, 0b10010, "sqxtun", int_aarch64_neon_sqxtun>;
3022 defm SUQADD : SIMDTwoVectorBHSDTied<0, 0b00011, "suqadd",int_aarch64_neon_suqadd>;
3023 defm UADALP : SIMDLongTwoVectorTied<1, 0b00110, "uadalp",
3024 BinOpFrag<(add node:$LHS, (int_aarch64_neon_uaddlp node:$RHS))> >;
3025 defm UADDLP : SIMDLongTwoVector<1, 0b00010, "uaddlp",
3026 int_aarch64_neon_uaddlp>;
3027 defm UCVTF : SIMDTwoVectorIntToFP<1, 0, 0b11101, "ucvtf", uint_to_fp>;
3028 defm UQXTN : SIMDMixedTwoVector<1, 0b10100, "uqxtn", int_aarch64_neon_uqxtn>;
3029 defm URECPE : SIMDTwoVectorS<0, 1, 0b11100, "urecpe", int_aarch64_neon_urecpe>;
3030 defm URSQRTE: SIMDTwoVectorS<1, 1, 0b11100, "ursqrte", int_aarch64_neon_ursqrte>;
3031 defm USQADD : SIMDTwoVectorBHSDTied<1, 0b00011, "usqadd",int_aarch64_neon_usqadd>;
3032 defm XTN : SIMDMixedTwoVector<0, 0b10010, "xtn", trunc>;
3034 def : Pat<(v4f16 (AArch64rev32 V64:$Rn)), (REV32v4i16 V64:$Rn)>;
3035 def : Pat<(v4f16 (AArch64rev64 V64:$Rn)), (REV64v4i16 V64:$Rn)>;
3036 def : Pat<(v8f16 (AArch64rev32 V128:$Rn)), (REV32v8i16 V128:$Rn)>;
3037 def : Pat<(v8f16 (AArch64rev64 V128:$Rn)), (REV64v8i16 V128:$Rn)>;
3038 def : Pat<(v2f32 (AArch64rev64 V64:$Rn)), (REV64v2i32 V64:$Rn)>;
3039 def : Pat<(v4f32 (AArch64rev64 V128:$Rn)), (REV64v4i32 V128:$Rn)>;
3041 // Patterns for vector long shift (by element width). These need to match all
3042 // three of zext, sext and anyext so it's easier to pull the patterns out of the
3044 multiclass SIMDVectorLShiftLongBySizeBHSPats<SDPatternOperator ext> {
3045 def : Pat<(AArch64vshl (v8i16 (ext (v8i8 V64:$Rn))), (i32 8)),
3046 (SHLLv8i8 V64:$Rn)>;
3047 def : Pat<(AArch64vshl (v8i16 (ext (extract_high_v16i8 V128:$Rn))), (i32 8)),
3048 (SHLLv16i8 V128:$Rn)>;
3049 def : Pat<(AArch64vshl (v4i32 (ext (v4i16 V64:$Rn))), (i32 16)),
3050 (SHLLv4i16 V64:$Rn)>;
3051 def : Pat<(AArch64vshl (v4i32 (ext (extract_high_v8i16 V128:$Rn))), (i32 16)),
3052 (SHLLv8i16 V128:$Rn)>;
3053 def : Pat<(AArch64vshl (v2i64 (ext (v2i32 V64:$Rn))), (i32 32)),
3054 (SHLLv2i32 V64:$Rn)>;
3055 def : Pat<(AArch64vshl (v2i64 (ext (extract_high_v4i32 V128:$Rn))), (i32 32)),
3056 (SHLLv4i32 V128:$Rn)>;
3059 defm : SIMDVectorLShiftLongBySizeBHSPats<anyext>;
3060 defm : SIMDVectorLShiftLongBySizeBHSPats<zext>;
3061 defm : SIMDVectorLShiftLongBySizeBHSPats<sext>;
3063 //===----------------------------------------------------------------------===//
3064 // Advanced SIMD three vector instructions.
3065 //===----------------------------------------------------------------------===//
3067 defm ADD : SIMDThreeSameVector<0, 0b10000, "add", add>;
3068 defm ADDP : SIMDThreeSameVector<0, 0b10111, "addp", int_aarch64_neon_addp>;
3069 defm CMEQ : SIMDThreeSameVector<1, 0b10001, "cmeq", AArch64cmeq>;
3070 defm CMGE : SIMDThreeSameVector<0, 0b00111, "cmge", AArch64cmge>;
3071 defm CMGT : SIMDThreeSameVector<0, 0b00110, "cmgt", AArch64cmgt>;
3072 defm CMHI : SIMDThreeSameVector<1, 0b00110, "cmhi", AArch64cmhi>;
3073 defm CMHS : SIMDThreeSameVector<1, 0b00111, "cmhs", AArch64cmhs>;
3074 defm CMTST : SIMDThreeSameVector<0, 0b10001, "cmtst", AArch64cmtst>;
3075 defm FABD : SIMDThreeSameVectorFP<1,1,0b010,"fabd", int_aarch64_neon_fabd>;
3076 defm FACGE : SIMDThreeSameVectorFPCmp<1,0,0b101,"facge",int_aarch64_neon_facge>;
3077 defm FACGT : SIMDThreeSameVectorFPCmp<1,1,0b101,"facgt",int_aarch64_neon_facgt>;
3078 defm FADDP : SIMDThreeSameVectorFP<1,0,0b010,"faddp",int_aarch64_neon_addp>;
3079 defm FADD : SIMDThreeSameVectorFP<0,0,0b010,"fadd", fadd>;
3080 defm FCMEQ : SIMDThreeSameVectorFPCmp<0, 0, 0b100, "fcmeq", AArch64fcmeq>;
3081 defm FCMGE : SIMDThreeSameVectorFPCmp<1, 0, 0b100, "fcmge", AArch64fcmge>;
3082 defm FCMGT : SIMDThreeSameVectorFPCmp<1, 1, 0b100, "fcmgt", AArch64fcmgt>;
3083 defm FDIV : SIMDThreeSameVectorFP<1,0,0b111,"fdiv", fdiv>;
3084 defm FMAXNMP : SIMDThreeSameVectorFP<1,0,0b000,"fmaxnmp", int_aarch64_neon_fmaxnmp>;
3085 defm FMAXNM : SIMDThreeSameVectorFP<0,0,0b000,"fmaxnm", fmaxnum>;
3086 defm FMAXP : SIMDThreeSameVectorFP<1,0,0b110,"fmaxp", int_aarch64_neon_fmaxp>;
3087 defm FMAX : SIMDThreeSameVectorFP<0,0,0b110,"fmax", fmaxnan>;
3088 defm FMINNMP : SIMDThreeSameVectorFP<1,1,0b000,"fminnmp", int_aarch64_neon_fminnmp>;
3089 defm FMINNM : SIMDThreeSameVectorFP<0,1,0b000,"fminnm", fminnum>;
3090 defm FMINP : SIMDThreeSameVectorFP<1,1,0b110,"fminp", int_aarch64_neon_fminp>;
3091 defm FMIN : SIMDThreeSameVectorFP<0,1,0b110,"fmin", fminnan>;
3093 // NOTE: The operands of the PatFrag are reordered on FMLA/FMLS because the
3094 // instruction expects the addend first, while the fma intrinsic puts it last.
3095 defm FMLA : SIMDThreeSameVectorFPTied<0, 0, 0b001, "fmla",
3096 TriOpFrag<(fma node:$RHS, node:$MHS, node:$LHS)> >;
3097 defm FMLS : SIMDThreeSameVectorFPTied<0, 1, 0b001, "fmls",
3098 TriOpFrag<(fma node:$MHS, (fneg node:$RHS), node:$LHS)> >;
3100 // The following def pats catch the case where the LHS of an FMA is negated.
3101 // The TriOpFrag above catches the case where the middle operand is negated.
3102 def : Pat<(v2f32 (fma (fneg V64:$Rn), V64:$Rm, V64:$Rd)),
3103 (FMLSv2f32 V64:$Rd, V64:$Rn, V64:$Rm)>;
3105 def : Pat<(v4f32 (fma (fneg V128:$Rn), V128:$Rm, V128:$Rd)),
3106 (FMLSv4f32 V128:$Rd, V128:$Rn, V128:$Rm)>;
3108 def : Pat<(v2f64 (fma (fneg V128:$Rn), V128:$Rm, V128:$Rd)),
3109 (FMLSv2f64 V128:$Rd, V128:$Rn, V128:$Rm)>;
3111 defm FMULX : SIMDThreeSameVectorFP<0,0,0b011,"fmulx", int_aarch64_neon_fmulx>;
3112 defm FMUL : SIMDThreeSameVectorFP<1,0,0b011,"fmul", fmul>;
3113 defm FRECPS : SIMDThreeSameVectorFP<0,0,0b111,"frecps", int_aarch64_neon_frecps>;
3114 defm FRSQRTS : SIMDThreeSameVectorFP<0,1,0b111,"frsqrts", int_aarch64_neon_frsqrts>;
3115 defm FSUB : SIMDThreeSameVectorFP<0,1,0b010,"fsub", fsub>;
3116 defm MLA : SIMDThreeSameVectorBHSTied<0, 0b10010, "mla",
3117 TriOpFrag<(add node:$LHS, (mul node:$MHS, node:$RHS))> >;
3118 defm MLS : SIMDThreeSameVectorBHSTied<1, 0b10010, "mls",
3119 TriOpFrag<(sub node:$LHS, (mul node:$MHS, node:$RHS))> >;
3120 defm MUL : SIMDThreeSameVectorBHS<0, 0b10011, "mul", mul>;
3121 defm PMUL : SIMDThreeSameVectorB<1, 0b10011, "pmul", int_aarch64_neon_pmul>;
3122 defm SABA : SIMDThreeSameVectorBHSTied<0, 0b01111, "saba",
3123 TriOpFrag<(add node:$LHS, (int_aarch64_neon_sabd node:$MHS, node:$RHS))> >;
3124 defm SABD : SIMDThreeSameVectorBHS<0,0b01110,"sabd", int_aarch64_neon_sabd>;
3125 defm SHADD : SIMDThreeSameVectorBHS<0,0b00000,"shadd", int_aarch64_neon_shadd>;
3126 defm SHSUB : SIMDThreeSameVectorBHS<0,0b00100,"shsub", int_aarch64_neon_shsub>;
3127 defm SMAXP : SIMDThreeSameVectorBHS<0,0b10100,"smaxp", int_aarch64_neon_smaxp>;
3128 defm SMAX : SIMDThreeSameVectorBHS<0,0b01100,"smax", smax>;
3129 defm SMINP : SIMDThreeSameVectorBHS<0,0b10101,"sminp", int_aarch64_neon_sminp>;
3130 defm SMIN : SIMDThreeSameVectorBHS<0,0b01101,"smin", smin>;
3131 defm SQADD : SIMDThreeSameVector<0,0b00001,"sqadd", int_aarch64_neon_sqadd>;
3132 defm SQDMULH : SIMDThreeSameVectorHS<0,0b10110,"sqdmulh",int_aarch64_neon_sqdmulh>;
3133 defm SQRDMULH : SIMDThreeSameVectorHS<1,0b10110,"sqrdmulh",int_aarch64_neon_sqrdmulh>;
3134 defm SQRSHL : SIMDThreeSameVector<0,0b01011,"sqrshl", int_aarch64_neon_sqrshl>;
3135 defm SQSHL : SIMDThreeSameVector<0,0b01001,"sqshl", int_aarch64_neon_sqshl>;
3136 defm SQSUB : SIMDThreeSameVector<0,0b00101,"sqsub", int_aarch64_neon_sqsub>;
3137 defm SRHADD : SIMDThreeSameVectorBHS<0,0b00010,"srhadd",int_aarch64_neon_srhadd>;
3138 defm SRSHL : SIMDThreeSameVector<0,0b01010,"srshl", int_aarch64_neon_srshl>;
3139 defm SSHL : SIMDThreeSameVector<0,0b01000,"sshl", int_aarch64_neon_sshl>;
3140 defm SUB : SIMDThreeSameVector<1,0b10000,"sub", sub>;
3141 defm UABA : SIMDThreeSameVectorBHSTied<1, 0b01111, "uaba",
3142 TriOpFrag<(add node:$LHS, (int_aarch64_neon_uabd node:$MHS, node:$RHS))> >;
3143 defm UABD : SIMDThreeSameVectorBHS<1,0b01110,"uabd", int_aarch64_neon_uabd>;
3144 defm UHADD : SIMDThreeSameVectorBHS<1,0b00000,"uhadd", int_aarch64_neon_uhadd>;
3145 defm UHSUB : SIMDThreeSameVectorBHS<1,0b00100,"uhsub", int_aarch64_neon_uhsub>;
3146 defm UMAXP : SIMDThreeSameVectorBHS<1,0b10100,"umaxp", int_aarch64_neon_umaxp>;
3147 defm UMAX : SIMDThreeSameVectorBHS<1,0b01100,"umax", umax>;
3148 defm UMINP : SIMDThreeSameVectorBHS<1,0b10101,"uminp", int_aarch64_neon_uminp>;
3149 defm UMIN : SIMDThreeSameVectorBHS<1,0b01101,"umin", umin>;
3150 defm UQADD : SIMDThreeSameVector<1,0b00001,"uqadd", int_aarch64_neon_uqadd>;
3151 defm UQRSHL : SIMDThreeSameVector<1,0b01011,"uqrshl", int_aarch64_neon_uqrshl>;
3152 defm UQSHL : SIMDThreeSameVector<1,0b01001,"uqshl", int_aarch64_neon_uqshl>;
3153 defm UQSUB : SIMDThreeSameVector<1,0b00101,"uqsub", int_aarch64_neon_uqsub>;
3154 defm URHADD : SIMDThreeSameVectorBHS<1,0b00010,"urhadd", int_aarch64_neon_urhadd>;
3155 defm URSHL : SIMDThreeSameVector<1,0b01010,"urshl", int_aarch64_neon_urshl>;
3156 defm USHL : SIMDThreeSameVector<1,0b01000,"ushl", int_aarch64_neon_ushl>;
3157 defm SQRDMLAH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10000,"sqrdmlah",
3158 int_aarch64_neon_sqadd>;
3159 defm SQRDMLSH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10001,"sqrdmlsh",
3160 int_aarch64_neon_sqsub>;
3162 defm AND : SIMDLogicalThreeVector<0, 0b00, "and", and>;
3163 defm BIC : SIMDLogicalThreeVector<0, 0b01, "bic",
3164 BinOpFrag<(and node:$LHS, (vnot node:$RHS))> >;
3165 defm BIF : SIMDLogicalThreeVector<1, 0b11, "bif">;
3166 defm BIT : SIMDLogicalThreeVectorTied<1, 0b10, "bit", AArch64bit>;
3167 defm BSL : SIMDLogicalThreeVectorTied<1, 0b01, "bsl",
3168 TriOpFrag<(or (and node:$LHS, node:$MHS), (and (vnot node:$LHS), node:$RHS))>>;
3169 defm EOR : SIMDLogicalThreeVector<1, 0b00, "eor", xor>;
3170 defm ORN : SIMDLogicalThreeVector<0, 0b11, "orn",
3171 BinOpFrag<(or node:$LHS, (vnot node:$RHS))> >;
3172 defm ORR : SIMDLogicalThreeVector<0, 0b10, "orr", or>;
3175 def : Pat<(AArch64bsl (v8i8 V64:$Rd), V64:$Rn, V64:$Rm),
3176 (BSLv8i8 V64:$Rd, V64:$Rn, V64:$Rm)>;
3177 def : Pat<(AArch64bsl (v4i16 V64:$Rd), V64:$Rn, V64:$Rm),
3178 (BSLv8i8 V64:$Rd, V64:$Rn, V64:$Rm)>;
3179 def : Pat<(AArch64bsl (v2i32 V64:$Rd), V64:$Rn, V64:$Rm),
3180 (BSLv8i8 V64:$Rd, V64:$Rn, V64:$Rm)>;
3181 def : Pat<(AArch64bsl (v1i64 V64:$Rd), V64:$Rn, V64:$Rm),
3182 (BSLv8i8 V64:$Rd, V64:$Rn, V64:$Rm)>;
3184 def : Pat<(AArch64bsl (v16i8 V128:$Rd), V128:$Rn, V128:$Rm),
3185 (BSLv16i8 V128:$Rd, V128:$Rn, V128:$Rm)>;
3186 def : Pat<(AArch64bsl (v8i16 V128:$Rd), V128:$Rn, V128:$Rm),
3187 (BSLv16i8 V128:$Rd, V128:$Rn, V128:$Rm)>;
3188 def : Pat<(AArch64bsl (v4i32 V128:$Rd), V128:$Rn, V128:$Rm),
3189 (BSLv16i8 V128:$Rd, V128:$Rn, V128:$Rm)>;
3190 def : Pat<(AArch64bsl (v2i64 V128:$Rd), V128:$Rn, V128:$Rm),
3191 (BSLv16i8 V128:$Rd, V128:$Rn, V128:$Rm)>;
3193 def : InstAlias<"mov{\t$dst.16b, $src.16b|.16b\t$dst, $src}",
3194 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 1>;
3195 def : InstAlias<"mov{\t$dst.8h, $src.8h|.8h\t$dst, $src}",
3196 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 0>;
3197 def : InstAlias<"mov{\t$dst.4s, $src.4s|.4s\t$dst, $src}",
3198 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 0>;
3199 def : InstAlias<"mov{\t$dst.2d, $src.2d|.2d\t$dst, $src}",
3200 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 0>;
3202 def : InstAlias<"mov{\t$dst.8b, $src.8b|.8b\t$dst, $src}",
3203 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 1>;
3204 def : InstAlias<"mov{\t$dst.4h, $src.4h|.4h\t$dst, $src}",
3205 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 0>;
3206 def : InstAlias<"mov{\t$dst.2s, $src.2s|.2s\t$dst, $src}",
3207 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 0>;
3208 def : InstAlias<"mov{\t$dst.1d, $src.1d|.1d\t$dst, $src}",
3209 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 0>;
3211 def : InstAlias<"{cmls\t$dst.8b, $src1.8b, $src2.8b" #
3212 "|cmls.8b\t$dst, $src1, $src2}",
3213 (CMHSv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
3214 def : InstAlias<"{cmls\t$dst.16b, $src1.16b, $src2.16b" #
3215 "|cmls.16b\t$dst, $src1, $src2}",
3216 (CMHSv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
3217 def : InstAlias<"{cmls\t$dst.4h, $src1.4h, $src2.4h" #
3218 "|cmls.4h\t$dst, $src1, $src2}",
3219 (CMHSv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
3220 def : InstAlias<"{cmls\t$dst.8h, $src1.8h, $src2.8h" #
3221 "|cmls.8h\t$dst, $src1, $src2}",
3222 (CMHSv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
3223 def : InstAlias<"{cmls\t$dst.2s, $src1.2s, $src2.2s" #
3224 "|cmls.2s\t$dst, $src1, $src2}",
3225 (CMHSv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
3226 def : InstAlias<"{cmls\t$dst.4s, $src1.4s, $src2.4s" #
3227 "|cmls.4s\t$dst, $src1, $src2}",
3228 (CMHSv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
3229 def : InstAlias<"{cmls\t$dst.2d, $src1.2d, $src2.2d" #
3230 "|cmls.2d\t$dst, $src1, $src2}",
3231 (CMHSv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
3233 def : InstAlias<"{cmlo\t$dst.8b, $src1.8b, $src2.8b" #
3234 "|cmlo.8b\t$dst, $src1, $src2}",
3235 (CMHIv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
3236 def : InstAlias<"{cmlo\t$dst.16b, $src1.16b, $src2.16b" #
3237 "|cmlo.16b\t$dst, $src1, $src2}",
3238 (CMHIv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
3239 def : InstAlias<"{cmlo\t$dst.4h, $src1.4h, $src2.4h" #
3240 "|cmlo.4h\t$dst, $src1, $src2}",
3241 (CMHIv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
3242 def : InstAlias<"{cmlo\t$dst.8h, $src1.8h, $src2.8h" #
3243 "|cmlo.8h\t$dst, $src1, $src2}",
3244 (CMHIv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
3245 def : InstAlias<"{cmlo\t$dst.2s, $src1.2s, $src2.2s" #
3246 "|cmlo.2s\t$dst, $src1, $src2}",
3247 (CMHIv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
3248 def : InstAlias<"{cmlo\t$dst.4s, $src1.4s, $src2.4s" #
3249 "|cmlo.4s\t$dst, $src1, $src2}",
3250 (CMHIv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
3251 def : InstAlias<"{cmlo\t$dst.2d, $src1.2d, $src2.2d" #
3252 "|cmlo.2d\t$dst, $src1, $src2}",
3253 (CMHIv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
3255 def : InstAlias<"{cmle\t$dst.8b, $src1.8b, $src2.8b" #
3256 "|cmle.8b\t$dst, $src1, $src2}",
3257 (CMGEv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
3258 def : InstAlias<"{cmle\t$dst.16b, $src1.16b, $src2.16b" #
3259 "|cmle.16b\t$dst, $src1, $src2}",
3260 (CMGEv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
3261 def : InstAlias<"{cmle\t$dst.4h, $src1.4h, $src2.4h" #
3262 "|cmle.4h\t$dst, $src1, $src2}",
3263 (CMGEv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
3264 def : InstAlias<"{cmle\t$dst.8h, $src1.8h, $src2.8h" #
3265 "|cmle.8h\t$dst, $src1, $src2}",
3266 (CMGEv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
3267 def : InstAlias<"{cmle\t$dst.2s, $src1.2s, $src2.2s" #
3268 "|cmle.2s\t$dst, $src1, $src2}",
3269 (CMGEv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
3270 def : InstAlias<"{cmle\t$dst.4s, $src1.4s, $src2.4s" #
3271 "|cmle.4s\t$dst, $src1, $src2}",
3272 (CMGEv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
3273 def : InstAlias<"{cmle\t$dst.2d, $src1.2d, $src2.2d" #
3274 "|cmle.2d\t$dst, $src1, $src2}",
3275 (CMGEv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
3277 def : InstAlias<"{cmlt\t$dst.8b, $src1.8b, $src2.8b" #
3278 "|cmlt.8b\t$dst, $src1, $src2}",
3279 (CMGTv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
3280 def : InstAlias<"{cmlt\t$dst.16b, $src1.16b, $src2.16b" #
3281 "|cmlt.16b\t$dst, $src1, $src2}",
3282 (CMGTv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
3283 def : InstAlias<"{cmlt\t$dst.4h, $src1.4h, $src2.4h" #
3284 "|cmlt.4h\t$dst, $src1, $src2}",
3285 (CMGTv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
3286 def : InstAlias<"{cmlt\t$dst.8h, $src1.8h, $src2.8h" #
3287 "|cmlt.8h\t$dst, $src1, $src2}",
3288 (CMGTv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
3289 def : InstAlias<"{cmlt\t$dst.2s, $src1.2s, $src2.2s" #
3290 "|cmlt.2s\t$dst, $src1, $src2}",
3291 (CMGTv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
3292 def : InstAlias<"{cmlt\t$dst.4s, $src1.4s, $src2.4s" #
3293 "|cmlt.4s\t$dst, $src1, $src2}",
3294 (CMGTv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
3295 def : InstAlias<"{cmlt\t$dst.2d, $src1.2d, $src2.2d" #
3296 "|cmlt.2d\t$dst, $src1, $src2}",
3297 (CMGTv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
3299 let Predicates = [HasNEON, HasFullFP16] in {
3300 def : InstAlias<"{fcmle\t$dst.4h, $src1.4h, $src2.4h" #
3301 "|fcmle.4h\t$dst, $src1, $src2}",
3302 (FCMGEv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
3303 def : InstAlias<"{fcmle\t$dst.8h, $src1.8h, $src2.8h" #
3304 "|fcmle.8h\t$dst, $src1, $src2}",
3305 (FCMGEv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
3307 def : InstAlias<"{fcmle\t$dst.2s, $src1.2s, $src2.2s" #
3308 "|fcmle.2s\t$dst, $src1, $src2}",
3309 (FCMGEv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
3310 def : InstAlias<"{fcmle\t$dst.4s, $src1.4s, $src2.4s" #
3311 "|fcmle.4s\t$dst, $src1, $src2}",
3312 (FCMGEv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
3313 def : InstAlias<"{fcmle\t$dst.2d, $src1.2d, $src2.2d" #
3314 "|fcmle.2d\t$dst, $src1, $src2}",
3315 (FCMGEv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
3317 let Predicates = [HasNEON, HasFullFP16] in {
3318 def : InstAlias<"{fcmlt\t$dst.4h, $src1.4h, $src2.4h" #
3319 "|fcmlt.4h\t$dst, $src1, $src2}",
3320 (FCMGTv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
3321 def : InstAlias<"{fcmlt\t$dst.8h, $src1.8h, $src2.8h" #
3322 "|fcmlt.8h\t$dst, $src1, $src2}",
3323 (FCMGTv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
3325 def : InstAlias<"{fcmlt\t$dst.2s, $src1.2s, $src2.2s" #
3326 "|fcmlt.2s\t$dst, $src1, $src2}",
3327 (FCMGTv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
3328 def : InstAlias<"{fcmlt\t$dst.4s, $src1.4s, $src2.4s" #
3329 "|fcmlt.4s\t$dst, $src1, $src2}",
3330 (FCMGTv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
3331 def : InstAlias<"{fcmlt\t$dst.2d, $src1.2d, $src2.2d" #
3332 "|fcmlt.2d\t$dst, $src1, $src2}",
3333 (FCMGTv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
3335 let Predicates = [HasNEON, HasFullFP16] in {
3336 def : InstAlias<"{facle\t$dst.4h, $src1.4h, $src2.4h" #
3337 "|facle.4h\t$dst, $src1, $src2}",
3338 (FACGEv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
3339 def : InstAlias<"{facle\t$dst.8h, $src1.8h, $src2.8h" #
3340 "|facle.8h\t$dst, $src1, $src2}",
3341 (FACGEv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
3343 def : InstAlias<"{facle\t$dst.2s, $src1.2s, $src2.2s" #
3344 "|facle.2s\t$dst, $src1, $src2}",
3345 (FACGEv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
3346 def : InstAlias<"{facle\t$dst.4s, $src1.4s, $src2.4s" #
3347 "|facle.4s\t$dst, $src1, $src2}",
3348 (FACGEv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
3349 def : InstAlias<"{facle\t$dst.2d, $src1.2d, $src2.2d" #
3350 "|facle.2d\t$dst, $src1, $src2}",
3351 (FACGEv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
3353 let Predicates = [HasNEON, HasFullFP16] in {
3354 def : InstAlias<"{faclt\t$dst.4h, $src1.4h, $src2.4h" #
3355 "|faclt.4h\t$dst, $src1, $src2}",
3356 (FACGTv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
3357 def : InstAlias<"{faclt\t$dst.8h, $src1.8h, $src2.8h" #
3358 "|faclt.8h\t$dst, $src1, $src2}",
3359 (FACGTv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
3361 def : InstAlias<"{faclt\t$dst.2s, $src1.2s, $src2.2s" #
3362 "|faclt.2s\t$dst, $src1, $src2}",
3363 (FACGTv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
3364 def : InstAlias<"{faclt\t$dst.4s, $src1.4s, $src2.4s" #
3365 "|faclt.4s\t$dst, $src1, $src2}",
3366 (FACGTv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
3367 def : InstAlias<"{faclt\t$dst.2d, $src1.2d, $src2.2d" #
3368 "|faclt.2d\t$dst, $src1, $src2}",
3369 (FACGTv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
3371 //===----------------------------------------------------------------------===//
3372 // Advanced SIMD three scalar instructions.
3373 //===----------------------------------------------------------------------===//
3375 defm ADD : SIMDThreeScalarD<0, 0b10000, "add", add>;
3376 defm CMEQ : SIMDThreeScalarD<1, 0b10001, "cmeq", AArch64cmeq>;
3377 defm CMGE : SIMDThreeScalarD<0, 0b00111, "cmge", AArch64cmge>;
3378 defm CMGT : SIMDThreeScalarD<0, 0b00110, "cmgt", AArch64cmgt>;
3379 defm CMHI : SIMDThreeScalarD<1, 0b00110, "cmhi", AArch64cmhi>;
3380 defm CMHS : SIMDThreeScalarD<1, 0b00111, "cmhs", AArch64cmhs>;
3381 defm CMTST : SIMDThreeScalarD<0, 0b10001, "cmtst", AArch64cmtst>;
3382 defm FABD : SIMDFPThreeScalar<1, 1, 0b010, "fabd", int_aarch64_sisd_fabd>;
3383 def : Pat<(v1f64 (int_aarch64_neon_fabd (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
3384 (FABD64 FPR64:$Rn, FPR64:$Rm)>;
3385 defm FACGE : SIMDThreeScalarFPCmp<1, 0, 0b101, "facge",
3386 int_aarch64_neon_facge>;
3387 defm FACGT : SIMDThreeScalarFPCmp<1, 1, 0b101, "facgt",
3388 int_aarch64_neon_facgt>;
3389 defm FCMEQ : SIMDThreeScalarFPCmp<0, 0, 0b100, "fcmeq", AArch64fcmeq>;
3390 defm FCMGE : SIMDThreeScalarFPCmp<1, 0, 0b100, "fcmge", AArch64fcmge>;
3391 defm FCMGT : SIMDThreeScalarFPCmp<1, 1, 0b100, "fcmgt", AArch64fcmgt>;
3392 defm FMULX : SIMDFPThreeScalar<0, 0, 0b011, "fmulx", int_aarch64_neon_fmulx>;
3393 defm FRECPS : SIMDFPThreeScalar<0, 0, 0b111, "frecps", int_aarch64_neon_frecps>;
3394 defm FRSQRTS : SIMDFPThreeScalar<0, 1, 0b111, "frsqrts", int_aarch64_neon_frsqrts>;
3395 defm SQADD : SIMDThreeScalarBHSD<0, 0b00001, "sqadd", int_aarch64_neon_sqadd>;
3396 defm SQDMULH : SIMDThreeScalarHS< 0, 0b10110, "sqdmulh", int_aarch64_neon_sqdmulh>;
3397 defm SQRDMULH : SIMDThreeScalarHS< 1, 0b10110, "sqrdmulh", int_aarch64_neon_sqrdmulh>;
3398 defm SQRSHL : SIMDThreeScalarBHSD<0, 0b01011, "sqrshl",int_aarch64_neon_sqrshl>;
3399 defm SQSHL : SIMDThreeScalarBHSD<0, 0b01001, "sqshl", int_aarch64_neon_sqshl>;
3400 defm SQSUB : SIMDThreeScalarBHSD<0, 0b00101, "sqsub", int_aarch64_neon_sqsub>;
3401 defm SRSHL : SIMDThreeScalarD< 0, 0b01010, "srshl", int_aarch64_neon_srshl>;
3402 defm SSHL : SIMDThreeScalarD< 0, 0b01000, "sshl", int_aarch64_neon_sshl>;
3403 defm SUB : SIMDThreeScalarD< 1, 0b10000, "sub", sub>;
3404 defm UQADD : SIMDThreeScalarBHSD<1, 0b00001, "uqadd", int_aarch64_neon_uqadd>;
3405 defm UQRSHL : SIMDThreeScalarBHSD<1, 0b01011, "uqrshl",int_aarch64_neon_uqrshl>;
3406 defm UQSHL : SIMDThreeScalarBHSD<1, 0b01001, "uqshl", int_aarch64_neon_uqshl>;
3407 defm UQSUB : SIMDThreeScalarBHSD<1, 0b00101, "uqsub", int_aarch64_neon_uqsub>;
3408 defm URSHL : SIMDThreeScalarD< 1, 0b01010, "urshl", int_aarch64_neon_urshl>;
3409 defm USHL : SIMDThreeScalarD< 1, 0b01000, "ushl", int_aarch64_neon_ushl>;
3410 let Predicates = [HasRDM] in {
3411 defm SQRDMLAH : SIMDThreeScalarHSTied<1, 0, 0b10000, "sqrdmlah">;
3412 defm SQRDMLSH : SIMDThreeScalarHSTied<1, 0, 0b10001, "sqrdmlsh">;
3413 def : Pat<(i32 (int_aarch64_neon_sqadd
3415 (i32 (int_aarch64_neon_sqrdmulh (i32 FPR32:$Rn),
3416 (i32 FPR32:$Rm))))),
3417 (SQRDMLAHv1i32 FPR32:$Rd, FPR32:$Rn, FPR32:$Rm)>;
3418 def : Pat<(i32 (int_aarch64_neon_sqsub
3420 (i32 (int_aarch64_neon_sqrdmulh (i32 FPR32:$Rn),
3421 (i32 FPR32:$Rm))))),
3422 (SQRDMLSHv1i32 FPR32:$Rd, FPR32:$Rn, FPR32:$Rm)>;
3425 def : InstAlias<"cmls $dst, $src1, $src2",
3426 (CMHSv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3427 def : InstAlias<"cmle $dst, $src1, $src2",
3428 (CMGEv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3429 def : InstAlias<"cmlo $dst, $src1, $src2",
3430 (CMHIv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3431 def : InstAlias<"cmlt $dst, $src1, $src2",
3432 (CMGTv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3433 def : InstAlias<"fcmle $dst, $src1, $src2",
3434 (FCMGE32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
3435 def : InstAlias<"fcmle $dst, $src1, $src2",
3436 (FCMGE64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3437 def : InstAlias<"fcmlt $dst, $src1, $src2",
3438 (FCMGT32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
3439 def : InstAlias<"fcmlt $dst, $src1, $src2",
3440 (FCMGT64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3441 def : InstAlias<"facle $dst, $src1, $src2",
3442 (FACGE32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
3443 def : InstAlias<"facle $dst, $src1, $src2",
3444 (FACGE64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3445 def : InstAlias<"faclt $dst, $src1, $src2",
3446 (FACGT32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
3447 def : InstAlias<"faclt $dst, $src1, $src2",
3448 (FACGT64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
3450 //===----------------------------------------------------------------------===//
3451 // Advanced SIMD three scalar instructions (mixed operands).
3452 //===----------------------------------------------------------------------===//
3453 defm SQDMULL : SIMDThreeScalarMixedHS<0, 0b11010, "sqdmull",
3454 int_aarch64_neon_sqdmulls_scalar>;
3455 defm SQDMLAL : SIMDThreeScalarMixedTiedHS<0, 0b10010, "sqdmlal">;
3456 defm SQDMLSL : SIMDThreeScalarMixedTiedHS<0, 0b10110, "sqdmlsl">;
3458 def : Pat<(i64 (int_aarch64_neon_sqadd (i64 FPR64:$Rd),
3459 (i64 (int_aarch64_neon_sqdmulls_scalar (i32 FPR32:$Rn),
3460 (i32 FPR32:$Rm))))),
3461 (SQDMLALi32 FPR64:$Rd, FPR32:$Rn, FPR32:$Rm)>;
3462 def : Pat<(i64 (int_aarch64_neon_sqsub (i64 FPR64:$Rd),
3463 (i64 (int_aarch64_neon_sqdmulls_scalar (i32 FPR32:$Rn),
3464 (i32 FPR32:$Rm))))),
3465 (SQDMLSLi32 FPR64:$Rd, FPR32:$Rn, FPR32:$Rm)>;
3467 //===----------------------------------------------------------------------===//
3468 // Advanced SIMD two scalar instructions.
3469 //===----------------------------------------------------------------------===//
3471 defm ABS : SIMDTwoScalarD< 0, 0b01011, "abs", abs>;
3472 defm CMEQ : SIMDCmpTwoScalarD< 0, 0b01001, "cmeq", AArch64cmeqz>;
3473 defm CMGE : SIMDCmpTwoScalarD< 1, 0b01000, "cmge", AArch64cmgez>;
3474 defm CMGT : SIMDCmpTwoScalarD< 0, 0b01000, "cmgt", AArch64cmgtz>;
3475 defm CMLE : SIMDCmpTwoScalarD< 1, 0b01001, "cmle", AArch64cmlez>;
3476 defm CMLT : SIMDCmpTwoScalarD< 0, 0b01010, "cmlt", AArch64cmltz>;
3477 defm FCMEQ : SIMDFPCmpTwoScalar<0, 1, 0b01101, "fcmeq", AArch64fcmeqz>;
3478 defm FCMGE : SIMDFPCmpTwoScalar<1, 1, 0b01100, "fcmge", AArch64fcmgez>;
3479 defm FCMGT : SIMDFPCmpTwoScalar<0, 1, 0b01100, "fcmgt", AArch64fcmgtz>;
3480 defm FCMLE : SIMDFPCmpTwoScalar<1, 1, 0b01101, "fcmle", AArch64fcmlez>;
3481 defm FCMLT : SIMDFPCmpTwoScalar<0, 1, 0b01110, "fcmlt", AArch64fcmltz>;
3482 defm FCVTAS : SIMDFPTwoScalar< 0, 0, 0b11100, "fcvtas">;
3483 defm FCVTAU : SIMDFPTwoScalar< 1, 0, 0b11100, "fcvtau">;
3484 defm FCVTMS : SIMDFPTwoScalar< 0, 0, 0b11011, "fcvtms">;
3485 defm FCVTMU : SIMDFPTwoScalar< 1, 0, 0b11011, "fcvtmu">;
3486 defm FCVTNS : SIMDFPTwoScalar< 0, 0, 0b11010, "fcvtns">;
3487 defm FCVTNU : SIMDFPTwoScalar< 1, 0, 0b11010, "fcvtnu">;
3488 defm FCVTPS : SIMDFPTwoScalar< 0, 1, 0b11010, "fcvtps">;
3489 defm FCVTPU : SIMDFPTwoScalar< 1, 1, 0b11010, "fcvtpu">;
3490 def FCVTXNv1i64 : SIMDInexactCvtTwoScalar<0b10110, "fcvtxn">;
3491 defm FCVTZS : SIMDFPTwoScalar< 0, 1, 0b11011, "fcvtzs">;
3492 defm FCVTZU : SIMDFPTwoScalar< 1, 1, 0b11011, "fcvtzu">;
3493 defm FRECPE : SIMDFPTwoScalar< 0, 1, 0b11101, "frecpe">;
3494 defm FRECPX : SIMDFPTwoScalar< 0, 1, 0b11111, "frecpx">;
3495 defm FRSQRTE : SIMDFPTwoScalar< 1, 1, 0b11101, "frsqrte">;
3496 defm NEG : SIMDTwoScalarD< 1, 0b01011, "neg",
3497 UnOpFrag<(sub immAllZerosV, node:$LHS)> >;
3498 defm SCVTF : SIMDFPTwoScalarCVT< 0, 0, 0b11101, "scvtf", AArch64sitof>;
3499 defm SQABS : SIMDTwoScalarBHSD< 0, 0b00111, "sqabs", int_aarch64_neon_sqabs>;
3500 defm SQNEG : SIMDTwoScalarBHSD< 1, 0b00111, "sqneg", int_aarch64_neon_sqneg>;
3501 defm SQXTN : SIMDTwoScalarMixedBHS< 0, 0b10100, "sqxtn", int_aarch64_neon_scalar_sqxtn>;
3502 defm SQXTUN : SIMDTwoScalarMixedBHS< 1, 0b10010, "sqxtun", int_aarch64_neon_scalar_sqxtun>;
3503 defm SUQADD : SIMDTwoScalarBHSDTied< 0, 0b00011, "suqadd",
3504 int_aarch64_neon_suqadd>;
3505 defm UCVTF : SIMDFPTwoScalarCVT< 1, 0, 0b11101, "ucvtf", AArch64uitof>;
3506 defm UQXTN : SIMDTwoScalarMixedBHS<1, 0b10100, "uqxtn", int_aarch64_neon_scalar_uqxtn>;
3507 defm USQADD : SIMDTwoScalarBHSDTied< 1, 0b00011, "usqadd",
3508 int_aarch64_neon_usqadd>;
3510 def : Pat<(AArch64neg (v1i64 V64:$Rn)), (NEGv1i64 V64:$Rn)>;
3512 def : Pat<(v1i64 (int_aarch64_neon_fcvtas (v1f64 FPR64:$Rn))),
3513 (FCVTASv1i64 FPR64:$Rn)>;
3514 def : Pat<(v1i64 (int_aarch64_neon_fcvtau (v1f64 FPR64:$Rn))),
3515 (FCVTAUv1i64 FPR64:$Rn)>;
3516 def : Pat<(v1i64 (int_aarch64_neon_fcvtms (v1f64 FPR64:$Rn))),
3517 (FCVTMSv1i64 FPR64:$Rn)>;
3518 def : Pat<(v1i64 (int_aarch64_neon_fcvtmu (v1f64 FPR64:$Rn))),
3519 (FCVTMUv1i64 FPR64:$Rn)>;
3520 def : Pat<(v1i64 (int_aarch64_neon_fcvtns (v1f64 FPR64:$Rn))),
3521 (FCVTNSv1i64 FPR64:$Rn)>;
3522 def : Pat<(v1i64 (int_aarch64_neon_fcvtnu (v1f64 FPR64:$Rn))),
3523 (FCVTNUv1i64 FPR64:$Rn)>;
3524 def : Pat<(v1i64 (int_aarch64_neon_fcvtps (v1f64 FPR64:$Rn))),
3525 (FCVTPSv1i64 FPR64:$Rn)>;
3526 def : Pat<(v1i64 (int_aarch64_neon_fcvtpu (v1f64 FPR64:$Rn))),
3527 (FCVTPUv1i64 FPR64:$Rn)>;
3529 def : Pat<(f32 (int_aarch64_neon_frecpe (f32 FPR32:$Rn))),
3530 (FRECPEv1i32 FPR32:$Rn)>;
3531 def : Pat<(f64 (int_aarch64_neon_frecpe (f64 FPR64:$Rn))),
3532 (FRECPEv1i64 FPR64:$Rn)>;
3533 def : Pat<(v1f64 (int_aarch64_neon_frecpe (v1f64 FPR64:$Rn))),
3534 (FRECPEv1i64 FPR64:$Rn)>;
3536 def : Pat<(f32 (AArch64frecpe (f32 FPR32:$Rn))),
3537 (FRECPEv1i32 FPR32:$Rn)>;
3538 def : Pat<(v2f32 (AArch64frecpe (v2f32 V64:$Rn))),
3539 (FRECPEv2f32 V64:$Rn)>;
3540 def : Pat<(v4f32 (AArch64frecpe (v4f32 FPR128:$Rn))),
3541 (FRECPEv4f32 FPR128:$Rn)>;
3542 def : Pat<(f64 (AArch64frecpe (f64 FPR64:$Rn))),
3543 (FRECPEv1i64 FPR64:$Rn)>;
3544 def : Pat<(v1f64 (AArch64frecpe (v1f64 FPR64:$Rn))),
3545 (FRECPEv1i64 FPR64:$Rn)>;
3546 def : Pat<(v2f64 (AArch64frecpe (v2f64 FPR128:$Rn))),
3547 (FRECPEv2f64 FPR128:$Rn)>;
3549 def : Pat<(f32 (AArch64frecps (f32 FPR32:$Rn), (f32 FPR32:$Rm))),
3550 (FRECPS32 FPR32:$Rn, FPR32:$Rm)>;
3551 def : Pat<(v2f32 (AArch64frecps (v2f32 V64:$Rn), (v2f32 V64:$Rm))),
3552 (FRECPSv2f32 V64:$Rn, V64:$Rm)>;
3553 def : Pat<(v4f32 (AArch64frecps (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm))),
3554 (FRECPSv4f32 FPR128:$Rn, FPR128:$Rm)>;
3555 def : Pat<(f64 (AArch64frecps (f64 FPR64:$Rn), (f64 FPR64:$Rm))),
3556 (FRECPS64 FPR64:$Rn, FPR64:$Rm)>;
3557 def : Pat<(v2f64 (AArch64frecps (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))),
3558 (FRECPSv2f64 FPR128:$Rn, FPR128:$Rm)>;
3560 def : Pat<(f32 (int_aarch64_neon_frecpx (f32 FPR32:$Rn))),
3561 (FRECPXv1i32 FPR32:$Rn)>;
3562 def : Pat<(f64 (int_aarch64_neon_frecpx (f64 FPR64:$Rn))),
3563 (FRECPXv1i64 FPR64:$Rn)>;
3565 def : Pat<(f32 (int_aarch64_neon_frsqrte (f32 FPR32:$Rn))),
3566 (FRSQRTEv1i32 FPR32:$Rn)>;
3567 def : Pat<(f64 (int_aarch64_neon_frsqrte (f64 FPR64:$Rn))),
3568 (FRSQRTEv1i64 FPR64:$Rn)>;
3569 def : Pat<(v1f64 (int_aarch64_neon_frsqrte (v1f64 FPR64:$Rn))),
3570 (FRSQRTEv1i64 FPR64:$Rn)>;
3572 def : Pat<(f32 (AArch64frsqrte (f32 FPR32:$Rn))),
3573 (FRSQRTEv1i32 FPR32:$Rn)>;
3574 def : Pat<(v2f32 (AArch64frsqrte (v2f32 V64:$Rn))),
3575 (FRSQRTEv2f32 V64:$Rn)>;
3576 def : Pat<(v4f32 (AArch64frsqrte (v4f32 FPR128:$Rn))),
3577 (FRSQRTEv4f32 FPR128:$Rn)>;
3578 def : Pat<(f64 (AArch64frsqrte (f64 FPR64:$Rn))),
3579 (FRSQRTEv1i64 FPR64:$Rn)>;
3580 def : Pat<(v1f64 (AArch64frsqrte (v1f64 FPR64:$Rn))),
3581 (FRSQRTEv1i64 FPR64:$Rn)>;
3582 def : Pat<(v2f64 (AArch64frsqrte (v2f64 FPR128:$Rn))),
3583 (FRSQRTEv2f64 FPR128:$Rn)>;
3585 def : Pat<(f32 (AArch64frsqrts (f32 FPR32:$Rn), (f32 FPR32:$Rm))),
3586 (FRSQRTS32 FPR32:$Rn, FPR32:$Rm)>;
3587 def : Pat<(v2f32 (AArch64frsqrts (v2f32 V64:$Rn), (v2f32 V64:$Rm))),
3588 (FRSQRTSv2f32 V64:$Rn, V64:$Rm)>;
3589 def : Pat<(v4f32 (AArch64frsqrts (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm))),
3590 (FRSQRTSv4f32 FPR128:$Rn, FPR128:$Rm)>;
3591 def : Pat<(f64 (AArch64frsqrts (f64 FPR64:$Rn), (f64 FPR64:$Rm))),
3592 (FRSQRTS64 FPR64:$Rn, FPR64:$Rm)>;
3593 def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))),
3594 (FRSQRTSv2f64 FPR128:$Rn, FPR128:$Rm)>;
3596 // If an integer is about to be converted to a floating point value,
3597 // just load it on the floating point unit.
3598 // Here are the patterns for 8 and 16-bits to float.
3600 multiclass UIntToFPROLoadPat<ValueType DstTy, ValueType SrcTy,
3601 SDPatternOperator loadop, Instruction UCVTF,
3602 ROAddrMode ro, Instruction LDRW, Instruction LDRX,
3604 def : Pat<(DstTy (uint_to_fp (SrcTy
3605 (loadop (ro.Wpat GPR64sp:$Rn, GPR32:$Rm,
3606 ro.Wext:$extend))))),
3607 (UCVTF (INSERT_SUBREG (DstTy (IMPLICIT_DEF)),
3608 (LDRW GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend),
3611 def : Pat<(DstTy (uint_to_fp (SrcTy
3612 (loadop (ro.Xpat GPR64sp:$Rn, GPR64:$Rm,
3613 ro.Wext:$extend))))),
3614 (UCVTF (INSERT_SUBREG (DstTy (IMPLICIT_DEF)),
3615 (LDRX GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend),
3619 defm : UIntToFPROLoadPat<f32, i32, zextloadi8,
3620 UCVTFv1i32, ro8, LDRBroW, LDRBroX, bsub>;
3621 def : Pat <(f32 (uint_to_fp (i32
3622 (zextloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))))),
3623 (UCVTFv1i32 (INSERT_SUBREG (f32 (IMPLICIT_DEF)),
3624 (LDRBui GPR64sp:$Rn, uimm12s1:$offset), bsub))>;
3625 def : Pat <(f32 (uint_to_fp (i32
3626 (zextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))))),
3627 (UCVTFv1i32 (INSERT_SUBREG (f32 (IMPLICIT_DEF)),
3628 (LDURBi GPR64sp:$Rn, simm9:$offset), bsub))>;
3629 // 16-bits -> float.
3630 defm : UIntToFPROLoadPat<f32, i32, zextloadi16,
3631 UCVTFv1i32, ro16, LDRHroW, LDRHroX, hsub>;
3632 def : Pat <(f32 (uint_to_fp (i32
3633 (zextloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))))),
3634 (UCVTFv1i32 (INSERT_SUBREG (f32 (IMPLICIT_DEF)),
3635 (LDRHui GPR64sp:$Rn, uimm12s2:$offset), hsub))>;
3636 def : Pat <(f32 (uint_to_fp (i32
3637 (zextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))))),
3638 (UCVTFv1i32 (INSERT_SUBREG (f32 (IMPLICIT_DEF)),
3639 (LDURHi GPR64sp:$Rn, simm9:$offset), hsub))>;
3640 // 32-bits are handled in target specific dag combine:
3641 // performIntToFpCombine.
3642 // 64-bits integer to 32-bits floating point, not possible with
3643 // UCVTF on floating point registers (both source and destination
3644 // must have the same size).
3646 // Here are the patterns for 8, 16, 32, and 64-bits to double.
3647 // 8-bits -> double.
3648 defm : UIntToFPROLoadPat<f64, i32, zextloadi8,
3649 UCVTFv1i64, ro8, LDRBroW, LDRBroX, bsub>;
3650 def : Pat <(f64 (uint_to_fp (i32
3651 (zextloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))))),
3652 (UCVTFv1i64 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
3653 (LDRBui GPR64sp:$Rn, uimm12s1:$offset), bsub))>;
3654 def : Pat <(f64 (uint_to_fp (i32
3655 (zextloadi8 (am_unscaled8 GPR64sp:$Rn, simm9:$offset))))),
3656 (UCVTFv1i64 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
3657 (LDURBi GPR64sp:$Rn, simm9:$offset), bsub))>;
3658 // 16-bits -> double.
3659 defm : UIntToFPROLoadPat<f64, i32, zextloadi16,
3660 UCVTFv1i64, ro16, LDRHroW, LDRHroX, hsub>;
3661 def : Pat <(f64 (uint_to_fp (i32
3662 (zextloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))))),
3663 (UCVTFv1i64 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
3664 (LDRHui GPR64sp:$Rn, uimm12s2:$offset), hsub))>;
3665 def : Pat <(f64 (uint_to_fp (i32
3666 (zextloadi16 (am_unscaled16 GPR64sp:$Rn, simm9:$offset))))),
3667 (UCVTFv1i64 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
3668 (LDURHi GPR64sp:$Rn, simm9:$offset), hsub))>;
3669 // 32-bits -> double.
3670 defm : UIntToFPROLoadPat<f64, i32, load,
3671 UCVTFv1i64, ro32, LDRSroW, LDRSroX, ssub>;
3672 def : Pat <(f64 (uint_to_fp (i32
3673 (load (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))))),
3674 (UCVTFv1i64 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
3675 (LDRSui GPR64sp:$Rn, uimm12s4:$offset), ssub))>;
3676 def : Pat <(f64 (uint_to_fp (i32
3677 (load (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))),
3678 (UCVTFv1i64 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
3679 (LDURSi GPR64sp:$Rn, simm9:$offset), ssub))>;
3680 // 64-bits -> double are handled in target specific dag combine:
3681 // performIntToFpCombine.
3683 //===----------------------------------------------------------------------===//
3684 // Advanced SIMD three different-sized vector instructions.
3685 //===----------------------------------------------------------------------===//
3687 defm ADDHN : SIMDNarrowThreeVectorBHS<0,0b0100,"addhn", int_aarch64_neon_addhn>;
3688 defm SUBHN : SIMDNarrowThreeVectorBHS<0,0b0110,"subhn", int_aarch64_neon_subhn>;
3689 defm RADDHN : SIMDNarrowThreeVectorBHS<1,0b0100,"raddhn",int_aarch64_neon_raddhn>;
3690 defm RSUBHN : SIMDNarrowThreeVectorBHS<1,0b0110,"rsubhn",int_aarch64_neon_rsubhn>;
3691 defm PMULL : SIMDDifferentThreeVectorBD<0,0b1110,"pmull",int_aarch64_neon_pmull>;
3692 defm SABAL : SIMDLongThreeVectorTiedBHSabal<0,0b0101,"sabal",
3693 int_aarch64_neon_sabd>;
3694 defm SABDL : SIMDLongThreeVectorBHSabdl<0, 0b0111, "sabdl",
3695 int_aarch64_neon_sabd>;
3696 defm SADDL : SIMDLongThreeVectorBHS< 0, 0b0000, "saddl",
3697 BinOpFrag<(add (sext node:$LHS), (sext node:$RHS))>>;
3698 defm SADDW : SIMDWideThreeVectorBHS< 0, 0b0001, "saddw",
3699 BinOpFrag<(add node:$LHS, (sext node:$RHS))>>;
3700 defm SMLAL : SIMDLongThreeVectorTiedBHS<0, 0b1000, "smlal",
3701 TriOpFrag<(add node:$LHS, (int_aarch64_neon_smull node:$MHS, node:$RHS))>>;
3702 defm SMLSL : SIMDLongThreeVectorTiedBHS<0, 0b1010, "smlsl",
3703 TriOpFrag<(sub node:$LHS, (int_aarch64_neon_smull node:$MHS, node:$RHS))>>;
3704 defm SMULL : SIMDLongThreeVectorBHS<0, 0b1100, "smull", int_aarch64_neon_smull>;
3705 defm SQDMLAL : SIMDLongThreeVectorSQDMLXTiedHS<0, 0b1001, "sqdmlal",
3706 int_aarch64_neon_sqadd>;
3707 defm SQDMLSL : SIMDLongThreeVectorSQDMLXTiedHS<0, 0b1011, "sqdmlsl",
3708 int_aarch64_neon_sqsub>;
3709 defm SQDMULL : SIMDLongThreeVectorHS<0, 0b1101, "sqdmull",
3710 int_aarch64_neon_sqdmull>;
3711 defm SSUBL : SIMDLongThreeVectorBHS<0, 0b0010, "ssubl",
3712 BinOpFrag<(sub (sext node:$LHS), (sext node:$RHS))>>;
3713 defm SSUBW : SIMDWideThreeVectorBHS<0, 0b0011, "ssubw",
3714 BinOpFrag<(sub node:$LHS, (sext node:$RHS))>>;
3715 defm UABAL : SIMDLongThreeVectorTiedBHSabal<1, 0b0101, "uabal",
3716 int_aarch64_neon_uabd>;
3717 defm UADDL : SIMDLongThreeVectorBHS<1, 0b0000, "uaddl",
3718 BinOpFrag<(add (zext node:$LHS), (zext node:$RHS))>>;
3719 defm UADDW : SIMDWideThreeVectorBHS<1, 0b0001, "uaddw",
3720 BinOpFrag<(add node:$LHS, (zext node:$RHS))>>;
3721 defm UMLAL : SIMDLongThreeVectorTiedBHS<1, 0b1000, "umlal",
3722 TriOpFrag<(add node:$LHS, (int_aarch64_neon_umull node:$MHS, node:$RHS))>>;
3723 defm UMLSL : SIMDLongThreeVectorTiedBHS<1, 0b1010, "umlsl",
3724 TriOpFrag<(sub node:$LHS, (int_aarch64_neon_umull node:$MHS, node:$RHS))>>;
3725 defm UMULL : SIMDLongThreeVectorBHS<1, 0b1100, "umull", int_aarch64_neon_umull>;
3726 defm USUBL : SIMDLongThreeVectorBHS<1, 0b0010, "usubl",
3727 BinOpFrag<(sub (zext node:$LHS), (zext node:$RHS))>>;
3728 defm USUBW : SIMDWideThreeVectorBHS< 1, 0b0011, "usubw",
3729 BinOpFrag<(sub node:$LHS, (zext node:$RHS))>>;
3731 // Additional patterns for SMULL and UMULL
3732 multiclass Neon_mul_widen_patterns<SDPatternOperator opnode,
3733 Instruction INST8B, Instruction INST4H, Instruction INST2S> {
3734 def : Pat<(v8i16 (opnode (v8i8 V64:$Rn), (v8i8 V64:$Rm))),
3735 (INST8B V64:$Rn, V64:$Rm)>;
3736 def : Pat<(v4i32 (opnode (v4i16 V64:$Rn), (v4i16 V64:$Rm))),
3737 (INST4H V64:$Rn, V64:$Rm)>;
3738 def : Pat<(v2i64 (opnode (v2i32 V64:$Rn), (v2i32 V64:$Rm))),
3739 (INST2S V64:$Rn, V64:$Rm)>;
3742 defm : Neon_mul_widen_patterns<AArch64smull, SMULLv8i8_v8i16,
3743 SMULLv4i16_v4i32, SMULLv2i32_v2i64>;
3744 defm : Neon_mul_widen_patterns<AArch64umull, UMULLv8i8_v8i16,
3745 UMULLv4i16_v4i32, UMULLv2i32_v2i64>;
3747 // Additional patterns for SMLAL/SMLSL and UMLAL/UMLSL
3748 multiclass Neon_mulacc_widen_patterns<SDPatternOperator opnode,
3749 Instruction INST8B, Instruction INST4H, Instruction INST2S> {
3750 def : Pat<(v8i16 (opnode (v8i16 V128:$Rd), (v8i8 V64:$Rn), (v8i8 V64:$Rm))),
3751 (INST8B V128:$Rd, V64:$Rn, V64:$Rm)>;
3752 def : Pat<(v4i32 (opnode (v4i32 V128:$Rd), (v4i16 V64:$Rn), (v4i16 V64:$Rm))),
3753 (INST4H V128:$Rd, V64:$Rn, V64:$Rm)>;
3754 def : Pat<(v2i64 (opnode (v2i64 V128:$Rd), (v2i32 V64:$Rn), (v2i32 V64:$Rm))),
3755 (INST2S V128:$Rd, V64:$Rn, V64:$Rm)>;
3758 defm : Neon_mulacc_widen_patterns<
3759 TriOpFrag<(add node:$LHS, (AArch64smull node:$MHS, node:$RHS))>,
3760 SMLALv8i8_v8i16, SMLALv4i16_v4i32, SMLALv2i32_v2i64>;
3761 defm : Neon_mulacc_widen_patterns<
3762 TriOpFrag<(add node:$LHS, (AArch64umull node:$MHS, node:$RHS))>,
3763 UMLALv8i8_v8i16, UMLALv4i16_v4i32, UMLALv2i32_v2i64>;
3764 defm : Neon_mulacc_widen_patterns<
3765 TriOpFrag<(sub node:$LHS, (AArch64smull node:$MHS, node:$RHS))>,
3766 SMLSLv8i8_v8i16, SMLSLv4i16_v4i32, SMLSLv2i32_v2i64>;
3767 defm : Neon_mulacc_widen_patterns<
3768 TriOpFrag<(sub node:$LHS, (AArch64umull node:$MHS, node:$RHS))>,
3769 UMLSLv8i8_v8i16, UMLSLv4i16_v4i32, UMLSLv2i32_v2i64>;
3771 // Patterns for 64-bit pmull
3772 def : Pat<(int_aarch64_neon_pmull64 V64:$Rn, V64:$Rm),
3773 (PMULLv1i64 V64:$Rn, V64:$Rm)>;
3774 def : Pat<(int_aarch64_neon_pmull64 (extractelt (v2i64 V128:$Rn), (i64 1)),
3775 (extractelt (v2i64 V128:$Rm), (i64 1))),
3776 (PMULLv2i64 V128:$Rn, V128:$Rm)>;
3778 // CodeGen patterns for addhn and subhn instructions, which can actually be
3779 // written in LLVM IR without too much difficulty.
3782 def : Pat<(v8i8 (trunc (v8i16 (AArch64vlshr (add V128:$Rn, V128:$Rm), (i32 8))))),
3783 (ADDHNv8i16_v8i8 V128:$Rn, V128:$Rm)>;
3784 def : Pat<(v4i16 (trunc (v4i32 (AArch64vlshr (add V128:$Rn, V128:$Rm),
3786 (ADDHNv4i32_v4i16 V128:$Rn, V128:$Rm)>;
3787 def : Pat<(v2i32 (trunc (v2i64 (AArch64vlshr (add V128:$Rn, V128:$Rm),
3789 (ADDHNv2i64_v2i32 V128:$Rn, V128:$Rm)>;
3790 def : Pat<(concat_vectors (v8i8 V64:$Rd),
3791 (trunc (v8i16 (AArch64vlshr (add V128:$Rn, V128:$Rm),
3793 (ADDHNv8i16_v16i8 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
3794 V128:$Rn, V128:$Rm)>;
3795 def : Pat<(concat_vectors (v4i16 V64:$Rd),
3796 (trunc (v4i32 (AArch64vlshr (add V128:$Rn, V128:$Rm),
3798 (ADDHNv4i32_v8i16 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
3799 V128:$Rn, V128:$Rm)>;
3800 def : Pat<(concat_vectors (v2i32 V64:$Rd),
3801 (trunc (v2i64 (AArch64vlshr (add V128:$Rn, V128:$Rm),
3803 (ADDHNv2i64_v4i32 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
3804 V128:$Rn, V128:$Rm)>;
3807 def : Pat<(v8i8 (trunc (v8i16 (AArch64vlshr (sub V128:$Rn, V128:$Rm), (i32 8))))),
3808 (SUBHNv8i16_v8i8 V128:$Rn, V128:$Rm)>;
3809 def : Pat<(v4i16 (trunc (v4i32 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
3811 (SUBHNv4i32_v4i16 V128:$Rn, V128:$Rm)>;
3812 def : Pat<(v2i32 (trunc (v2i64 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
3814 (SUBHNv2i64_v2i32 V128:$Rn, V128:$Rm)>;
3815 def : Pat<(concat_vectors (v8i8 V64:$Rd),
3816 (trunc (v8i16 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
3818 (SUBHNv8i16_v16i8 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
3819 V128:$Rn, V128:$Rm)>;
3820 def : Pat<(concat_vectors (v4i16 V64:$Rd),
3821 (trunc (v4i32 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
3823 (SUBHNv4i32_v8i16 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
3824 V128:$Rn, V128:$Rm)>;
3825 def : Pat<(concat_vectors (v2i32 V64:$Rd),
3826 (trunc (v2i64 (AArch64vlshr (sub V128:$Rn, V128:$Rm),
3828 (SUBHNv2i64_v4i32 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
3829 V128:$Rn, V128:$Rm)>;
3831 //----------------------------------------------------------------------------
3832 // AdvSIMD bitwise extract from vector instruction.
3833 //----------------------------------------------------------------------------
3835 defm EXT : SIMDBitwiseExtract<"ext">;
3837 def : Pat<(v4i16 (AArch64ext V64:$Rn, V64:$Rm, (i32 imm:$imm))),
3838 (EXTv8i8 V64:$Rn, V64:$Rm, imm:$imm)>;
3839 def : Pat<(v8i16 (AArch64ext V128:$Rn, V128:$Rm, (i32 imm:$imm))),
3840 (EXTv16i8 V128:$Rn, V128:$Rm, imm:$imm)>;
3841 def : Pat<(v2i32 (AArch64ext V64:$Rn, V64:$Rm, (i32 imm:$imm))),
3842 (EXTv8i8 V64:$Rn, V64:$Rm, imm:$imm)>;
3843 def : Pat<(v2f32 (AArch64ext V64:$Rn, V64:$Rm, (i32 imm:$imm))),
3844 (EXTv8i8 V64:$Rn, V64:$Rm, imm:$imm)>;
3845 def : Pat<(v4i32 (AArch64ext V128:$Rn, V128:$Rm, (i32 imm:$imm))),
3846 (EXTv16i8 V128:$Rn, V128:$Rm, imm:$imm)>;
3847 def : Pat<(v4f32 (AArch64ext V128:$Rn, V128:$Rm, (i32 imm:$imm))),
3848 (EXTv16i8 V128:$Rn, V128:$Rm, imm:$imm)>;
3849 def : Pat<(v2i64 (AArch64ext V128:$Rn, V128:$Rm, (i32 imm:$imm))),
3850 (EXTv16i8 V128:$Rn, V128:$Rm, imm:$imm)>;
3851 def : Pat<(v2f64 (AArch64ext V128:$Rn, V128:$Rm, (i32 imm:$imm))),
3852 (EXTv16i8 V128:$Rn, V128:$Rm, imm:$imm)>;
3853 def : Pat<(v4f16 (AArch64ext V64:$Rn, V64:$Rm, (i32 imm:$imm))),
3854 (EXTv8i8 V64:$Rn, V64:$Rm, imm:$imm)>;
3855 def : Pat<(v8f16 (AArch64ext V128:$Rn, V128:$Rm, (i32 imm:$imm))),
3856 (EXTv16i8 V128:$Rn, V128:$Rm, imm:$imm)>;
3858 // We use EXT to handle extract_subvector to copy the upper 64-bits of a
3860 def : Pat<(v8i8 (extract_subvector V128:$Rn, (i64 8))),
3861 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3862 def : Pat<(v4i16 (extract_subvector V128:$Rn, (i64 4))),
3863 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3864 def : Pat<(v2i32 (extract_subvector V128:$Rn, (i64 2))),
3865 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3866 def : Pat<(v1i64 (extract_subvector V128:$Rn, (i64 1))),
3867 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3868 def : Pat<(v4f16 (extract_subvector V128:$Rn, (i64 4))),
3869 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3870 def : Pat<(v2f32 (extract_subvector V128:$Rn, (i64 2))),
3871 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3872 def : Pat<(v1f64 (extract_subvector V128:$Rn, (i64 1))),
3873 (EXTRACT_SUBREG (EXTv16i8 V128:$Rn, V128:$Rn, 8), dsub)>;
3876 //----------------------------------------------------------------------------
3877 // AdvSIMD zip vector
3878 //----------------------------------------------------------------------------
3880 defm TRN1 : SIMDZipVector<0b010, "trn1", AArch64trn1>;
3881 defm TRN2 : SIMDZipVector<0b110, "trn2", AArch64trn2>;
3882 defm UZP1 : SIMDZipVector<0b001, "uzp1", AArch64uzp1>;
3883 defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>;
3884 defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>;
3885 defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>;
3887 //----------------------------------------------------------------------------
3888 // AdvSIMD TBL/TBX instructions
3889 //----------------------------------------------------------------------------
3891 defm TBL : SIMDTableLookup< 0, "tbl">;
3892 defm TBX : SIMDTableLookupTied<1, "tbx">;
3894 def : Pat<(v8i8 (int_aarch64_neon_tbl1 (v16i8 VecListOne128:$Rn), (v8i8 V64:$Ri))),
3895 (TBLv8i8One VecListOne128:$Rn, V64:$Ri)>;
3896 def : Pat<(v16i8 (int_aarch64_neon_tbl1 (v16i8 V128:$Ri), (v16i8 V128:$Rn))),
3897 (TBLv16i8One V128:$Ri, V128:$Rn)>;
3899 def : Pat<(v8i8 (int_aarch64_neon_tbx1 (v8i8 V64:$Rd),
3900 (v16i8 VecListOne128:$Rn), (v8i8 V64:$Ri))),
3901 (TBXv8i8One V64:$Rd, VecListOne128:$Rn, V64:$Ri)>;
3902 def : Pat<(v16i8 (int_aarch64_neon_tbx1 (v16i8 V128:$Rd),
3903 (v16i8 V128:$Ri), (v16i8 V128:$Rn))),
3904 (TBXv16i8One V128:$Rd, V128:$Ri, V128:$Rn)>;
3907 //----------------------------------------------------------------------------
3908 // AdvSIMD scalar CPY instruction
3909 //----------------------------------------------------------------------------
3911 defm CPY : SIMDScalarCPY<"cpy">;
3913 //----------------------------------------------------------------------------
3914 // AdvSIMD scalar pairwise instructions
3915 //----------------------------------------------------------------------------
3917 defm ADDP : SIMDPairwiseScalarD<0, 0b11011, "addp">;
3918 defm FADDP : SIMDFPPairwiseScalar<0, 0b01101, "faddp">;
3919 defm FMAXNMP : SIMDFPPairwiseScalar<0, 0b01100, "fmaxnmp">;
3920 defm FMAXP : SIMDFPPairwiseScalar<0, 0b01111, "fmaxp">;
3921 defm FMINNMP : SIMDFPPairwiseScalar<1, 0b01100, "fminnmp">;
3922 defm FMINP : SIMDFPPairwiseScalar<1, 0b01111, "fminp">;
3923 def : Pat<(v2i64 (AArch64saddv V128:$Rn)),
3924 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), (ADDPv2i64p V128:$Rn), dsub)>;
3925 def : Pat<(v2i64 (AArch64uaddv V128:$Rn)),
3926 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), (ADDPv2i64p V128:$Rn), dsub)>;
3927 def : Pat<(f32 (int_aarch64_neon_faddv (v2f32 V64:$Rn))),
3928 (FADDPv2i32p V64:$Rn)>;
3929 def : Pat<(f32 (int_aarch64_neon_faddv (v4f32 V128:$Rn))),
3930 (FADDPv2i32p (EXTRACT_SUBREG (FADDPv4f32 V128:$Rn, V128:$Rn), dsub))>;
3931 def : Pat<(f64 (int_aarch64_neon_faddv (v2f64 V128:$Rn))),
3932 (FADDPv2i64p V128:$Rn)>;
3933 def : Pat<(f32 (int_aarch64_neon_fmaxnmv (v2f32 V64:$Rn))),
3934 (FMAXNMPv2i32p V64:$Rn)>;
3935 def : Pat<(f64 (int_aarch64_neon_fmaxnmv (v2f64 V128:$Rn))),
3936 (FMAXNMPv2i64p V128:$Rn)>;
3937 def : Pat<(f32 (int_aarch64_neon_fmaxv (v2f32 V64:$Rn))),
3938 (FMAXPv2i32p V64:$Rn)>;
3939 def : Pat<(f64 (int_aarch64_neon_fmaxv (v2f64 V128:$Rn))),
3940 (FMAXPv2i64p V128:$Rn)>;
3941 def : Pat<(f32 (int_aarch64_neon_fminnmv (v2f32 V64:$Rn))),
3942 (FMINNMPv2i32p V64:$Rn)>;
3943 def : Pat<(f64 (int_aarch64_neon_fminnmv (v2f64 V128:$Rn))),
3944 (FMINNMPv2i64p V128:$Rn)>;
3945 def : Pat<(f32 (int_aarch64_neon_fminv (v2f32 V64:$Rn))),
3946 (FMINPv2i32p V64:$Rn)>;
3947 def : Pat<(f64 (int_aarch64_neon_fminv (v2f64 V128:$Rn))),
3948 (FMINPv2i64p V128:$Rn)>;
3950 //----------------------------------------------------------------------------
3951 // AdvSIMD INS/DUP instructions
3952 //----------------------------------------------------------------------------
3954 def DUPv8i8gpr : SIMDDupFromMain<0, {?,?,?,?,1}, ".8b", v8i8, V64, GPR32>;
3955 def DUPv16i8gpr : SIMDDupFromMain<1, {?,?,?,?,1}, ".16b", v16i8, V128, GPR32>;
3956 def DUPv4i16gpr : SIMDDupFromMain<0, {?,?,?,1,0}, ".4h", v4i16, V64, GPR32>;
3957 def DUPv8i16gpr : SIMDDupFromMain<1, {?,?,?,1,0}, ".8h", v8i16, V128, GPR32>;
3958 def DUPv2i32gpr : SIMDDupFromMain<0, {?,?,1,0,0}, ".2s", v2i32, V64, GPR32>;
3959 def DUPv4i32gpr : SIMDDupFromMain<1, {?,?,1,0,0}, ".4s", v4i32, V128, GPR32>;
3960 def DUPv2i64gpr : SIMDDupFromMain<1, {?,1,0,0,0}, ".2d", v2i64, V128, GPR64>;
3962 def DUPv2i64lane : SIMDDup64FromElement;
3963 def DUPv2i32lane : SIMDDup32FromElement<0, ".2s", v2i32, V64>;
3964 def DUPv4i32lane : SIMDDup32FromElement<1, ".4s", v4i32, V128>;
3965 def DUPv4i16lane : SIMDDup16FromElement<0, ".4h", v4i16, V64>;
3966 def DUPv8i16lane : SIMDDup16FromElement<1, ".8h", v8i16, V128>;
3967 def DUPv8i8lane : SIMDDup8FromElement <0, ".8b", v8i8, V64>;
3968 def DUPv16i8lane : SIMDDup8FromElement <1, ".16b", v16i8, V128>;
3970 def : Pat<(v2f32 (AArch64dup (f32 FPR32:$Rn))),
3971 (v2f32 (DUPv2i32lane
3972 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR32:$Rn, ssub),
3974 def : Pat<(v4f32 (AArch64dup (f32 FPR32:$Rn))),
3975 (v4f32 (DUPv4i32lane
3976 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR32:$Rn, ssub),
3978 def : Pat<(v2f64 (AArch64dup (f64 FPR64:$Rn))),
3979 (v2f64 (DUPv2i64lane
3980 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR64:$Rn, dsub),
3982 def : Pat<(v4f16 (AArch64dup (f16 FPR16:$Rn))),
3983 (v4f16 (DUPv4i16lane
3984 (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)), FPR16:$Rn, hsub),
3986 def : Pat<(v8f16 (AArch64dup (f16 FPR16:$Rn))),
3987 (v8f16 (DUPv8i16lane
3988 (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)), FPR16:$Rn, hsub),
3991 def : Pat<(v4f16 (AArch64duplane16 (v8f16 V128:$Rn), VectorIndexH:$imm)),
3992 (DUPv4i16lane V128:$Rn, VectorIndexH:$imm)>;
3993 def : Pat<(v8f16 (AArch64duplane16 (v8f16 V128:$Rn), VectorIndexH:$imm)),
3994 (DUPv8i16lane V128:$Rn, VectorIndexH:$imm)>;
3996 def : Pat<(v2f32 (AArch64duplane32 (v4f32 V128:$Rn), VectorIndexS:$imm)),
3997 (DUPv2i32lane V128:$Rn, VectorIndexS:$imm)>;
3998 def : Pat<(v4f32 (AArch64duplane32 (v4f32 V128:$Rn), VectorIndexS:$imm)),
3999 (DUPv4i32lane V128:$Rn, VectorIndexS:$imm)>;
4000 def : Pat<(v2f64 (AArch64duplane64 (v2f64 V128:$Rn), VectorIndexD:$imm)),
4001 (DUPv2i64lane V128:$Rn, VectorIndexD:$imm)>;
4003 // If there's an (AArch64dup (vector_extract ...) ...), we can use a duplane
4004 // instruction even if the types don't match: we just have to remap the lane
4005 // carefully. N.b. this trick only applies to truncations.
4006 def VecIndex_x2 : SDNodeXForm<imm, [{
4007 return CurDAG->getTargetConstant(2 * N->getZExtValue(), SDLoc(N), MVT::i64);
4009 def VecIndex_x4 : SDNodeXForm<imm, [{
4010 return CurDAG->getTargetConstant(4 * N->getZExtValue(), SDLoc(N), MVT::i64);
4012 def VecIndex_x8 : SDNodeXForm<imm, [{
4013 return CurDAG->getTargetConstant(8 * N->getZExtValue(), SDLoc(N), MVT::i64);
4016 multiclass DUPWithTruncPats<ValueType ResVT, ValueType Src64VT,
4017 ValueType Src128VT, ValueType ScalVT,
4018 Instruction DUP, SDNodeXForm IdxXFORM> {
4019 def : Pat<(ResVT (AArch64dup (ScalVT (vector_extract (Src128VT V128:$Rn),
4021 (DUP V128:$Rn, (IdxXFORM imm:$idx))>;
4023 def : Pat<(ResVT (AArch64dup (ScalVT (vector_extract (Src64VT V64:$Rn),
4025 (DUP (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), (IdxXFORM imm:$idx))>;
4028 defm : DUPWithTruncPats<v8i8, v4i16, v8i16, i32, DUPv8i8lane, VecIndex_x2>;
4029 defm : DUPWithTruncPats<v8i8, v2i32, v4i32, i32, DUPv8i8lane, VecIndex_x4>;
4030 defm : DUPWithTruncPats<v4i16, v2i32, v4i32, i32, DUPv4i16lane, VecIndex_x2>;
4032 defm : DUPWithTruncPats<v16i8, v4i16, v8i16, i32, DUPv16i8lane, VecIndex_x2>;
4033 defm : DUPWithTruncPats<v16i8, v2i32, v4i32, i32, DUPv16i8lane, VecIndex_x4>;
4034 defm : DUPWithTruncPats<v8i16, v2i32, v4i32, i32, DUPv8i16lane, VecIndex_x2>;
4036 multiclass DUPWithTrunci64Pats<ValueType ResVT, Instruction DUP,
4037 SDNodeXForm IdxXFORM> {
4038 def : Pat<(ResVT (AArch64dup (i32 (trunc (extractelt (v2i64 V128:$Rn),
4040 (DUP V128:$Rn, (IdxXFORM imm:$idx))>;
4042 def : Pat<(ResVT (AArch64dup (i32 (trunc (extractelt (v1i64 V64:$Rn),
4044 (DUP (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), (IdxXFORM imm:$idx))>;
4047 defm : DUPWithTrunci64Pats<v8i8, DUPv8i8lane, VecIndex_x8>;
4048 defm : DUPWithTrunci64Pats<v4i16, DUPv4i16lane, VecIndex_x4>;
4049 defm : DUPWithTrunci64Pats<v2i32, DUPv2i32lane, VecIndex_x2>;
4051 defm : DUPWithTrunci64Pats<v16i8, DUPv16i8lane, VecIndex_x8>;
4052 defm : DUPWithTrunci64Pats<v8i16, DUPv8i16lane, VecIndex_x4>;
4053 defm : DUPWithTrunci64Pats<v4i32, DUPv4i32lane, VecIndex_x2>;
4055 // SMOV and UMOV definitions, with some extra patterns for convenience
4059 def : Pat<(sext_inreg (vector_extract (v16i8 V128:$Rn), VectorIndexB:$idx), i8),
4060 (i32 (SMOVvi8to32 V128:$Rn, VectorIndexB:$idx))>;
4061 def : Pat<(sext_inreg (vector_extract (v16i8 V128:$Rn), VectorIndexB:$idx), i8),
4062 (i64 (SMOVvi8to64 V128:$Rn, VectorIndexB:$idx))>;
4063 def : Pat<(sext_inreg (vector_extract (v8i16 V128:$Rn), VectorIndexH:$idx),i16),
4064 (i32 (SMOVvi16to32 V128:$Rn, VectorIndexH:$idx))>;
4065 def : Pat<(sext_inreg (vector_extract (v8i16 V128:$Rn), VectorIndexH:$idx),i16),
4066 (i64 (SMOVvi16to64 V128:$Rn, VectorIndexH:$idx))>;
4067 def : Pat<(sext_inreg (vector_extract (v8i16 V128:$Rn), VectorIndexH:$idx),i16),
4068 (i32 (SMOVvi16to32 V128:$Rn, VectorIndexH:$idx))>;
4069 def : Pat<(sext (i32 (vector_extract (v4i32 V128:$Rn), VectorIndexS:$idx))),
4070 (i64 (SMOVvi32to64 V128:$Rn, VectorIndexS:$idx))>;
4072 def : Pat<(sext_inreg (i64 (anyext (i32 (vector_extract (v16i8 V128:$Rn),
4073 VectorIndexB:$idx)))), i8),
4074 (i64 (SMOVvi8to64 V128:$Rn, VectorIndexB:$idx))>;
4075 def : Pat<(sext_inreg (i64 (anyext (i32 (vector_extract (v8i16 V128:$Rn),
4076 VectorIndexH:$idx)))), i16),
4077 (i64 (SMOVvi16to64 V128:$Rn, VectorIndexH:$idx))>;
4079 // Extracting i8 or i16 elements will have the zero-extend transformed to
4080 // an 'and' mask by type legalization since neither i8 nor i16 are legal types
4081 // for AArch64. Match these patterns here since UMOV already zeroes out the high
4082 // bits of the destination register.
4083 def : Pat<(and (vector_extract (v16i8 V128:$Rn), VectorIndexB:$idx),
4085 (i32 (UMOVvi8 V128:$Rn, VectorIndexB:$idx))>;
4086 def : Pat<(and (vector_extract (v8i16 V128:$Rn), VectorIndexH:$idx),
4088 (i32 (UMOVvi16 V128:$Rn, VectorIndexH:$idx))>;
4092 def : Pat<(v16i8 (scalar_to_vector GPR32:$Rn)),
4093 (SUBREG_TO_REG (i32 0),
4094 (f32 (COPY_TO_REGCLASS GPR32:$Rn, FPR32)), ssub)>;
4095 def : Pat<(v8i8 (scalar_to_vector GPR32:$Rn)),
4096 (SUBREG_TO_REG (i32 0),
4097 (f32 (COPY_TO_REGCLASS GPR32:$Rn, FPR32)), ssub)>;
4099 def : Pat<(v8i16 (scalar_to_vector GPR32:$Rn)),
4100 (SUBREG_TO_REG (i32 0),
4101 (f32 (COPY_TO_REGCLASS GPR32:$Rn, FPR32)), ssub)>;
4102 def : Pat<(v4i16 (scalar_to_vector GPR32:$Rn)),
4103 (SUBREG_TO_REG (i32 0),
4104 (f32 (COPY_TO_REGCLASS GPR32:$Rn, FPR32)), ssub)>;
4106 def : Pat<(v2i32 (scalar_to_vector (i32 FPR32:$Rn))),
4107 (v2i32 (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)),
4108 (i32 FPR32:$Rn), ssub))>;
4109 def : Pat<(v4i32 (scalar_to_vector (i32 FPR32:$Rn))),
4110 (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
4111 (i32 FPR32:$Rn), ssub))>;
4112 def : Pat<(v2i64 (scalar_to_vector (i64 FPR64:$Rn))),
4113 (v2i64 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
4114 (i64 FPR64:$Rn), dsub))>;
4116 def : Pat<(v4f16 (scalar_to_vector (f16 FPR16:$Rn))),
4117 (INSERT_SUBREG (v4f16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
4118 def : Pat<(v8f16 (scalar_to_vector (f16 FPR16:$Rn))),
4119 (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
4121 def : Pat<(v4f32 (scalar_to_vector (f32 FPR32:$Rn))),
4122 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FPR32:$Rn, ssub)>;
4123 def : Pat<(v2f32 (scalar_to_vector (f32 FPR32:$Rn))),
4124 (INSERT_SUBREG (v2f32 (IMPLICIT_DEF)), FPR32:$Rn, ssub)>;
4125 def : Pat<(v2f64 (scalar_to_vector (f64 FPR64:$Rn))),
4126 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FPR64:$Rn, dsub)>;
4128 def : Pat<(v4f16 (vector_insert (v4f16 V64:$Rn),
4129 (f16 FPR16:$Rm), (i64 VectorIndexS:$imm))),
4132 (v8f16 (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), V64:$Rn, dsub)),
4134 (v8f16 (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), FPR16:$Rm, hsub)),
4138 def : Pat<(v8f16 (vector_insert (v8f16 V128:$Rn),
4139 (f16 FPR16:$Rm), (i64 VectorIndexH:$imm))),
4141 V128:$Rn, VectorIndexH:$imm,
4142 (v8f16 (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), FPR16:$Rm, hsub)),
4145 def : Pat<(v2f32 (vector_insert (v2f32 V64:$Rn),
4146 (f32 FPR32:$Rm), (i64 VectorIndexS:$imm))),
4149 (v4f32 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), V64:$Rn, dsub)),
4151 (v4f32 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FPR32:$Rm, ssub)),
4154 def : Pat<(v4f32 (vector_insert (v4f32 V128:$Rn),
4155 (f32 FPR32:$Rm), (i64 VectorIndexS:$imm))),
4157 V128:$Rn, VectorIndexS:$imm,
4158 (v4f32 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FPR32:$Rm, ssub)),
4160 def : Pat<(v2f64 (vector_insert (v2f64 V128:$Rn),
4161 (f64 FPR64:$Rm), (i64 VectorIndexD:$imm))),
4163 V128:$Rn, VectorIndexD:$imm,
4164 (v2f64 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FPR64:$Rm, dsub)),
4167 // Copy an element at a constant index in one vector into a constant indexed
4168 // element of another.
4169 // FIXME refactor to a shared class/dev parameterized on vector type, vector
4170 // index type and INS extension
4171 def : Pat<(v16i8 (int_aarch64_neon_vcopy_lane
4172 (v16i8 V128:$Vd), VectorIndexB:$idx, (v16i8 V128:$Vs),
4173 VectorIndexB:$idx2)),
4175 V128:$Vd, VectorIndexB:$idx, V128:$Vs, VectorIndexB:$idx2)
4177 def : Pat<(v8i16 (int_aarch64_neon_vcopy_lane
4178 (v8i16 V128:$Vd), VectorIndexH:$idx, (v8i16 V128:$Vs),
4179 VectorIndexH:$idx2)),
4181 V128:$Vd, VectorIndexH:$idx, V128:$Vs, VectorIndexH:$idx2)
4183 def : Pat<(v4i32 (int_aarch64_neon_vcopy_lane
4184 (v4i32 V128:$Vd), VectorIndexS:$idx, (v4i32 V128:$Vs),
4185 VectorIndexS:$idx2)),
4187 V128:$Vd, VectorIndexS:$idx, V128:$Vs, VectorIndexS:$idx2)
4189 def : Pat<(v2i64 (int_aarch64_neon_vcopy_lane
4190 (v2i64 V128:$Vd), VectorIndexD:$idx, (v2i64 V128:$Vs),
4191 VectorIndexD:$idx2)),
4193 V128:$Vd, VectorIndexD:$idx, V128:$Vs, VectorIndexD:$idx2)
4196 multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64,
4197 ValueType VTScal, Instruction INS> {
4198 def : Pat<(VT128 (vector_insert V128:$src,
4199 (VTScal (vector_extract (VT128 V128:$Rn), imm:$Immn)),
4201 (INS V128:$src, imm:$Immd, V128:$Rn, imm:$Immn)>;
4203 def : Pat<(VT128 (vector_insert V128:$src,
4204 (VTScal (vector_extract (VT64 V64:$Rn), imm:$Immn)),
4206 (INS V128:$src, imm:$Immd,
4207 (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn)>;
4209 def : Pat<(VT64 (vector_insert V64:$src,
4210 (VTScal (vector_extract (VT128 V128:$Rn), imm:$Immn)),
4212 (EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub),
4213 imm:$Immd, V128:$Rn, imm:$Immn),
4216 def : Pat<(VT64 (vector_insert V64:$src,
4217 (VTScal (vector_extract (VT64 V64:$Rn), imm:$Immn)),
4220 (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub), imm:$Immd,
4221 (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn),
4225 defm : Neon_INS_elt_pattern<v8f16, v4f16, f16, INSvi16lane>;
4226 defm : Neon_INS_elt_pattern<v4f32, v2f32, f32, INSvi32lane>;
4227 defm : Neon_INS_elt_pattern<v2f64, v1f64, f64, INSvi64lane>;
4230 // Floating point vector extractions are codegen'd as either a sequence of
4231 // subregister extractions, or a MOV (aka CPY here, alias for DUP) if
4232 // the lane number is anything other than zero.
4233 def : Pat<(vector_extract (v2f64 V128:$Rn), 0),
4234 (f64 (EXTRACT_SUBREG V128:$Rn, dsub))>;
4235 def : Pat<(vector_extract (v4f32 V128:$Rn), 0),
4236 (f32 (EXTRACT_SUBREG V128:$Rn, ssub))>;
4237 def : Pat<(vector_extract (v8f16 V128:$Rn), 0),
4238 (f16 (EXTRACT_SUBREG V128:$Rn, hsub))>;
4240 def : Pat<(vector_extract (v2f64 V128:$Rn), VectorIndexD:$idx),
4241 (f64 (CPYi64 V128:$Rn, VectorIndexD:$idx))>;
4242 def : Pat<(vector_extract (v4f32 V128:$Rn), VectorIndexS:$idx),
4243 (f32 (CPYi32 V128:$Rn, VectorIndexS:$idx))>;
4244 def : Pat<(vector_extract (v8f16 V128:$Rn), VectorIndexH:$idx),
4245 (f16 (CPYi16 V128:$Rn, VectorIndexH:$idx))>;
4247 // All concat_vectors operations are canonicalised to act on i64 vectors for
4248 // AArch64. In the general case we need an instruction, which had just as well be
4250 class ConcatPat<ValueType DstTy, ValueType SrcTy>
4251 : Pat<(DstTy (concat_vectors (SrcTy V64:$Rd), V64:$Rn)),
4252 (INSvi64lane (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), 1,
4253 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub), 0)>;
4255 def : ConcatPat<v2i64, v1i64>;
4256 def : ConcatPat<v2f64, v1f64>;
4257 def : ConcatPat<v4i32, v2i32>;
4258 def : ConcatPat<v4f32, v2f32>;
4259 def : ConcatPat<v8i16, v4i16>;
4260 def : ConcatPat<v8f16, v4f16>;
4261 def : ConcatPat<v16i8, v8i8>;
4263 // If the high lanes are undef, though, we can just ignore them:
4264 class ConcatUndefPat<ValueType DstTy, ValueType SrcTy>
4265 : Pat<(DstTy (concat_vectors (SrcTy V64:$Rn), undef)),
4266 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub)>;
4268 def : ConcatUndefPat<v2i64, v1i64>;
4269 def : ConcatUndefPat<v2f64, v1f64>;
4270 def : ConcatUndefPat<v4i32, v2i32>;
4271 def : ConcatUndefPat<v4f32, v2f32>;
4272 def : ConcatUndefPat<v8i16, v4i16>;
4273 def : ConcatUndefPat<v16i8, v8i8>;
4275 //----------------------------------------------------------------------------
4276 // AdvSIMD across lanes instructions
4277 //----------------------------------------------------------------------------
4279 defm ADDV : SIMDAcrossLanesBHS<0, 0b11011, "addv">;
4280 defm SMAXV : SIMDAcrossLanesBHS<0, 0b01010, "smaxv">;
4281 defm SMINV : SIMDAcrossLanesBHS<0, 0b11010, "sminv">;
4282 defm UMAXV : SIMDAcrossLanesBHS<1, 0b01010, "umaxv">;
4283 defm UMINV : SIMDAcrossLanesBHS<1, 0b11010, "uminv">;
4284 defm SADDLV : SIMDAcrossLanesHSD<0, 0b00011, "saddlv">;
4285 defm UADDLV : SIMDAcrossLanesHSD<1, 0b00011, "uaddlv">;
4286 defm FMAXNMV : SIMDFPAcrossLanes<0b01100, 0, "fmaxnmv", int_aarch64_neon_fmaxnmv>;
4287 defm FMAXV : SIMDFPAcrossLanes<0b01111, 0, "fmaxv", int_aarch64_neon_fmaxv>;
4288 defm FMINNMV : SIMDFPAcrossLanes<0b01100, 1, "fminnmv", int_aarch64_neon_fminnmv>;
4289 defm FMINV : SIMDFPAcrossLanes<0b01111, 1, "fminv", int_aarch64_neon_fminv>;
4291 // Patterns for across-vector intrinsics, that have a node equivalent, that
4292 // returns a vector (with only the low lane defined) instead of a scalar.
4293 // In effect, opNode is the same as (scalar_to_vector (IntNode)).
4294 multiclass SIMDAcrossLanesIntrinsic<string baseOpc,
4295 SDPatternOperator opNode> {
4296 // If a lane instruction caught the vector_extract around opNode, we can
4297 // directly match the latter to the instruction.
4298 def : Pat<(v8i8 (opNode V64:$Rn)),
4299 (INSERT_SUBREG (v8i8 (IMPLICIT_DEF)),
4300 (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), bsub)>;
4301 def : Pat<(v16i8 (opNode V128:$Rn)),
4302 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4303 (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), bsub)>;
4304 def : Pat<(v4i16 (opNode V64:$Rn)),
4305 (INSERT_SUBREG (v4i16 (IMPLICIT_DEF)),
4306 (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), hsub)>;
4307 def : Pat<(v8i16 (opNode V128:$Rn)),
4308 (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)),
4309 (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), hsub)>;
4310 def : Pat<(v4i32 (opNode V128:$Rn)),
4311 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
4312 (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn), ssub)>;
4315 // If none did, fallback to the explicit patterns, consuming the vector_extract.
4316 def : Pat<(i32 (vector_extract (insert_subvector undef, (v8i8 (opNode V64:$Rn)),
4317 (i32 0)), (i64 0))),
4318 (EXTRACT_SUBREG (INSERT_SUBREG (v8i8 (IMPLICIT_DEF)),
4319 (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn),
4321 def : Pat<(i32 (vector_extract (v16i8 (opNode V128:$Rn)), (i64 0))),
4322 (EXTRACT_SUBREG (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4323 (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn),
4325 def : Pat<(i32 (vector_extract (insert_subvector undef,
4326 (v4i16 (opNode V64:$Rn)), (i32 0)), (i64 0))),
4327 (EXTRACT_SUBREG (INSERT_SUBREG (v4i16 (IMPLICIT_DEF)),
4328 (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn),
4330 def : Pat<(i32 (vector_extract (v8i16 (opNode V128:$Rn)), (i64 0))),
4331 (EXTRACT_SUBREG (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)),
4332 (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn),
4334 def : Pat<(i32 (vector_extract (v4i32 (opNode V128:$Rn)), (i64 0))),
4335 (EXTRACT_SUBREG (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
4336 (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn),
4341 multiclass SIMDAcrossLanesSignedIntrinsic<string baseOpc,
4342 SDPatternOperator opNode>
4343 : SIMDAcrossLanesIntrinsic<baseOpc, opNode> {
4344 // If there is a sign extension after this intrinsic, consume it as smov already
4346 def : Pat<(i32 (sext_inreg (i32 (vector_extract (insert_subvector undef,
4347 (opNode (v8i8 V64:$Rn)), (i32 0)), (i64 0))), i8)),
4349 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4350 (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), bsub),
4352 def : Pat<(i32 (sext_inreg (i32 (vector_extract
4353 (opNode (v16i8 V128:$Rn)), (i64 0))), i8)),
4355 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4356 (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), bsub),
4358 def : Pat<(i32 (sext_inreg (i32 (vector_extract (insert_subvector undef,
4359 (opNode (v4i16 V64:$Rn)), (i32 0)), (i64 0))), i16)),
4361 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4362 (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), hsub),
4364 def : Pat<(i32 (sext_inreg (i32 (vector_extract
4365 (opNode (v8i16 V128:$Rn)), (i64 0))), i16)),
4367 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4368 (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), hsub),
4372 multiclass SIMDAcrossLanesUnsignedIntrinsic<string baseOpc,
4373 SDPatternOperator opNode>
4374 : SIMDAcrossLanesIntrinsic<baseOpc, opNode> {
4375 // If there is a masking operation keeping only what has been actually
4376 // generated, consume it.
4377 def : Pat<(i32 (and (i32 (vector_extract (insert_subvector undef,
4378 (opNode (v8i8 V64:$Rn)), (i32 0)), (i64 0))), maski8_or_more)),
4379 (i32 (EXTRACT_SUBREG
4380 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4381 (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), bsub),
4383 def : Pat<(i32 (and (i32 (vector_extract (opNode (v16i8 V128:$Rn)), (i64 0))),
4385 (i32 (EXTRACT_SUBREG
4386 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4387 (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), bsub),
4389 def : Pat<(i32 (and (i32 (vector_extract (insert_subvector undef,
4390 (opNode (v4i16 V64:$Rn)), (i32 0)), (i64 0))), maski16_or_more)),
4391 (i32 (EXTRACT_SUBREG
4392 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4393 (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), hsub),
4395 def : Pat<(i32 (and (i32 (vector_extract (opNode (v8i16 V128:$Rn)), (i64 0))),
4397 (i32 (EXTRACT_SUBREG
4398 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4399 (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), hsub),
4403 defm : SIMDAcrossLanesSignedIntrinsic<"ADDV", AArch64saddv>;
4404 // vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
4405 def : Pat<(v2i32 (AArch64saddv (v2i32 V64:$Rn))),
4406 (ADDPv2i32 V64:$Rn, V64:$Rn)>;
4408 defm : SIMDAcrossLanesUnsignedIntrinsic<"ADDV", AArch64uaddv>;
4409 // vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
4410 def : Pat<(v2i32 (AArch64uaddv (v2i32 V64:$Rn))),
4411 (ADDPv2i32 V64:$Rn, V64:$Rn)>;
4413 defm : SIMDAcrossLanesSignedIntrinsic<"SMAXV", AArch64smaxv>;
4414 def : Pat<(v2i32 (AArch64smaxv (v2i32 V64:$Rn))),
4415 (SMAXPv2i32 V64:$Rn, V64:$Rn)>;
4417 defm : SIMDAcrossLanesSignedIntrinsic<"SMINV", AArch64sminv>;
4418 def : Pat<(v2i32 (AArch64sminv (v2i32 V64:$Rn))),
4419 (SMINPv2i32 V64:$Rn, V64:$Rn)>;
4421 defm : SIMDAcrossLanesUnsignedIntrinsic<"UMAXV", AArch64umaxv>;
4422 def : Pat<(v2i32 (AArch64umaxv (v2i32 V64:$Rn))),
4423 (UMAXPv2i32 V64:$Rn, V64:$Rn)>;
4425 defm : SIMDAcrossLanesUnsignedIntrinsic<"UMINV", AArch64uminv>;
4426 def : Pat<(v2i32 (AArch64uminv (v2i32 V64:$Rn))),
4427 (UMINPv2i32 V64:$Rn, V64:$Rn)>;
4429 multiclass SIMDAcrossLanesSignedLongIntrinsic<string baseOpc, Intrinsic intOp> {
4430 def : Pat<(i32 (intOp (v8i8 V64:$Rn))),
4432 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4433 (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), hsub),
4435 def : Pat<(i32 (intOp (v16i8 V128:$Rn))),
4437 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4438 (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), hsub),
4441 def : Pat<(i32 (intOp (v4i16 V64:$Rn))),
4442 (i32 (EXTRACT_SUBREG
4443 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4444 (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), ssub),
4446 def : Pat<(i32 (intOp (v8i16 V128:$Rn))),
4447 (i32 (EXTRACT_SUBREG
4448 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4449 (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), ssub),
4452 def : Pat<(i64 (intOp (v4i32 V128:$Rn))),
4453 (i64 (EXTRACT_SUBREG
4454 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4455 (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn), dsub),
4459 multiclass SIMDAcrossLanesUnsignedLongIntrinsic<string baseOpc,
4461 def : Pat<(i32 (intOp (v8i8 V64:$Rn))),
4462 (i32 (EXTRACT_SUBREG
4463 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4464 (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), hsub),
4466 def : Pat<(i32 (intOp (v16i8 V128:$Rn))),
4467 (i32 (EXTRACT_SUBREG
4468 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4469 (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), hsub),
4472 def : Pat<(i32 (intOp (v4i16 V64:$Rn))),
4473 (i32 (EXTRACT_SUBREG
4474 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4475 (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), ssub),
4477 def : Pat<(i32 (intOp (v8i16 V128:$Rn))),
4478 (i32 (EXTRACT_SUBREG
4479 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4480 (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), ssub),
4483 def : Pat<(i64 (intOp (v4i32 V128:$Rn))),
4484 (i64 (EXTRACT_SUBREG
4485 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4486 (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn), dsub),
4490 defm : SIMDAcrossLanesSignedLongIntrinsic<"SADDLV", int_aarch64_neon_saddlv>;
4491 defm : SIMDAcrossLanesUnsignedLongIntrinsic<"UADDLV", int_aarch64_neon_uaddlv>;
4493 // The vaddlv_s32 intrinsic gets mapped to SADDLP.
4494 def : Pat<(i64 (int_aarch64_neon_saddlv (v2i32 V64:$Rn))),
4495 (i64 (EXTRACT_SUBREG
4496 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4497 (SADDLPv2i32_v1i64 V64:$Rn), dsub),
4499 // The vaddlv_u32 intrinsic gets mapped to UADDLP.
4500 def : Pat<(i64 (int_aarch64_neon_uaddlv (v2i32 V64:$Rn))),
4501 (i64 (EXTRACT_SUBREG
4502 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
4503 (UADDLPv2i32_v1i64 V64:$Rn), dsub),
4506 //------------------------------------------------------------------------------
4507 // AdvSIMD modified immediate instructions
4508 //------------------------------------------------------------------------------
4511 defm BIC : SIMDModifiedImmVectorShiftTied<1, 0b11, 0b01, "bic", AArch64bici>;
4513 defm ORR : SIMDModifiedImmVectorShiftTied<0, 0b11, 0b01, "orr", AArch64orri>;
4515 def : InstAlias<"bic $Vd.4h, $imm", (BICv4i16 V64:$Vd, imm0_255:$imm, 0)>;
4516 def : InstAlias<"bic $Vd.8h, $imm", (BICv8i16 V128:$Vd, imm0_255:$imm, 0)>;
4517 def : InstAlias<"bic $Vd.2s, $imm", (BICv2i32 V64:$Vd, imm0_255:$imm, 0)>;
4518 def : InstAlias<"bic $Vd.4s, $imm", (BICv4i32 V128:$Vd, imm0_255:$imm, 0)>;
4520 def : InstAlias<"bic.4h $Vd, $imm", (BICv4i16 V64:$Vd, imm0_255:$imm, 0)>;
4521 def : InstAlias<"bic.8h $Vd, $imm", (BICv8i16 V128:$Vd, imm0_255:$imm, 0)>;
4522 def : InstAlias<"bic.2s $Vd, $imm", (BICv2i32 V64:$Vd, imm0_255:$imm, 0)>;
4523 def : InstAlias<"bic.4s $Vd, $imm", (BICv4i32 V128:$Vd, imm0_255:$imm, 0)>;
4525 def : InstAlias<"orr $Vd.4h, $imm", (ORRv4i16 V64:$Vd, imm0_255:$imm, 0)>;
4526 def : InstAlias<"orr $Vd.8h, $imm", (ORRv8i16 V128:$Vd, imm0_255:$imm, 0)>;
4527 def : InstAlias<"orr $Vd.2s, $imm", (ORRv2i32 V64:$Vd, imm0_255:$imm, 0)>;
4528 def : InstAlias<"orr $Vd.4s, $imm", (ORRv4i32 V128:$Vd, imm0_255:$imm, 0)>;
4530 def : InstAlias<"orr.4h $Vd, $imm", (ORRv4i16 V64:$Vd, imm0_255:$imm, 0)>;
4531 def : InstAlias<"orr.8h $Vd, $imm", (ORRv8i16 V128:$Vd, imm0_255:$imm, 0)>;
4532 def : InstAlias<"orr.2s $Vd, $imm", (ORRv2i32 V64:$Vd, imm0_255:$imm, 0)>;
4533 def : InstAlias<"orr.4s $Vd, $imm", (ORRv4i32 V128:$Vd, imm0_255:$imm, 0)>;
4536 def FMOVv2f64_ns : SIMDModifiedImmVectorNoShift<1, 1, 0, 0b1111, V128, fpimm8,
4538 [(set (v2f64 V128:$Rd), (AArch64fmov imm0_255:$imm8))]>;
4539 def FMOVv2f32_ns : SIMDModifiedImmVectorNoShift<0, 0, 0, 0b1111, V64, fpimm8,
4541 [(set (v2f32 V64:$Rd), (AArch64fmov imm0_255:$imm8))]>;
4542 def FMOVv4f32_ns : SIMDModifiedImmVectorNoShift<1, 0, 0, 0b1111, V128, fpimm8,
4544 [(set (v4f32 V128:$Rd), (AArch64fmov imm0_255:$imm8))]>;
4545 let Predicates = [HasNEON, HasFullFP16] in {
4546 def FMOVv4f16_ns : SIMDModifiedImmVectorNoShift<0, 0, 1, 0b1111, V64, fpimm8,
4548 [(set (v4f16 V64:$Rd), (AArch64fmov imm0_255:$imm8))]>;
4549 def FMOVv8f16_ns : SIMDModifiedImmVectorNoShift<1, 0, 1, 0b1111, V128, fpimm8,
4551 [(set (v8f16 V128:$Rd), (AArch64fmov imm0_255:$imm8))]>;
4552 } // Predicates = [HasNEON, HasFullFP16]
4556 // EDIT byte mask: scalar
4557 let isReMaterializable = 1, isAsCheapAsAMove = 1 in
4558 def MOVID : SIMDModifiedImmScalarNoShift<0, 1, 0b1110, "movi",
4559 [(set FPR64:$Rd, simdimmtype10:$imm8)]>;
4560 // The movi_edit node has the immediate value already encoded, so we use
4561 // a plain imm0_255 here.
4562 def : Pat<(f64 (AArch64movi_edit imm0_255:$shift)),
4563 (MOVID imm0_255:$shift)>;
4565 def : Pat<(v1i64 immAllZerosV), (MOVID (i32 0))>;
4566 def : Pat<(v2i32 immAllZerosV), (MOVID (i32 0))>;
4567 def : Pat<(v4i16 immAllZerosV), (MOVID (i32 0))>;
4568 def : Pat<(v8i8 immAllZerosV), (MOVID (i32 0))>;
4570 def : Pat<(v1i64 immAllOnesV), (MOVID (i32 255))>;
4571 def : Pat<(v2i32 immAllOnesV), (MOVID (i32 255))>;
4572 def : Pat<(v4i16 immAllOnesV), (MOVID (i32 255))>;
4573 def : Pat<(v8i8 immAllOnesV), (MOVID (i32 255))>;
4575 // EDIT byte mask: 2d
4577 // The movi_edit node has the immediate value already encoded, so we use
4578 // a plain imm0_255 in the pattern
4579 let isReMaterializable = 1, isAsCheapAsAMove = 1 in
4580 def MOVIv2d_ns : SIMDModifiedImmVectorNoShift<1, 1, 0, 0b1110, V128,
4583 [(set (v2i64 V128:$Rd), (AArch64movi_edit imm0_255:$imm8))]>;
4585 def : Pat<(v2i64 immAllZerosV), (MOVIv2d_ns (i32 0))>;
4586 def : Pat<(v4i32 immAllZerosV), (MOVIv2d_ns (i32 0))>;
4587 def : Pat<(v8i16 immAllZerosV), (MOVIv2d_ns (i32 0))>;
4588 def : Pat<(v16i8 immAllZerosV), (MOVIv2d_ns (i32 0))>;
4590 def : Pat<(v2i64 immAllOnesV), (MOVIv2d_ns (i32 255))>;
4591 def : Pat<(v4i32 immAllOnesV), (MOVIv2d_ns (i32 255))>;
4592 def : Pat<(v8i16 immAllOnesV), (MOVIv2d_ns (i32 255))>;
4593 def : Pat<(v16i8 immAllOnesV), (MOVIv2d_ns (i32 255))>;
4595 def : Pat<(v2f64 (AArch64dup (f64 fpimm0))), (MOVIv2d_ns (i32 0))>;
4596 def : Pat<(v4f32 (AArch64dup (f32 fpimm0))), (MOVIv2d_ns (i32 0))>;
4598 // EDIT per word & halfword: 2s, 4h, 4s, & 8h
4599 defm MOVI : SIMDModifiedImmVectorShift<0, 0b10, 0b00, "movi">;
4601 def : InstAlias<"movi $Vd.4h, $imm", (MOVIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
4602 def : InstAlias<"movi $Vd.8h, $imm", (MOVIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
4603 def : InstAlias<"movi $Vd.2s, $imm", (MOVIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
4604 def : InstAlias<"movi $Vd.4s, $imm", (MOVIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
4606 def : InstAlias<"movi.4h $Vd, $imm", (MOVIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
4607 def : InstAlias<"movi.8h $Vd, $imm", (MOVIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
4608 def : InstAlias<"movi.2s $Vd, $imm", (MOVIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
4609 def : InstAlias<"movi.4s $Vd, $imm", (MOVIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
4611 def : Pat<(v2i32 (AArch64movi_shift imm0_255:$imm8, (i32 imm:$shift))),
4612 (MOVIv2i32 imm0_255:$imm8, imm:$shift)>;
4613 def : Pat<(v4i32 (AArch64movi_shift imm0_255:$imm8, (i32 imm:$shift))),
4614 (MOVIv4i32 imm0_255:$imm8, imm:$shift)>;
4615 def : Pat<(v4i16 (AArch64movi_shift imm0_255:$imm8, (i32 imm:$shift))),
4616 (MOVIv4i16 imm0_255:$imm8, imm:$shift)>;
4617 def : Pat<(v8i16 (AArch64movi_shift imm0_255:$imm8, (i32 imm:$shift))),
4618 (MOVIv8i16 imm0_255:$imm8, imm:$shift)>;
4620 // EDIT per word: 2s & 4s with MSL shifter
4621 def MOVIv2s_msl : SIMDModifiedImmMoveMSL<0, 0, {1,1,0,?}, V64, "movi", ".2s",
4622 [(set (v2i32 V64:$Rd),
4623 (AArch64movi_msl imm0_255:$imm8, (i32 imm:$shift)))]>;
4624 def MOVIv4s_msl : SIMDModifiedImmMoveMSL<1, 0, {1,1,0,?}, V128, "movi", ".4s",
4625 [(set (v4i32 V128:$Rd),
4626 (AArch64movi_msl imm0_255:$imm8, (i32 imm:$shift)))]>;
4628 // Per byte: 8b & 16b
4629 def MOVIv8b_ns : SIMDModifiedImmVectorNoShift<0, 0, 0, 0b1110, V64, imm0_255,
4631 [(set (v8i8 V64:$Rd), (AArch64movi imm0_255:$imm8))]>;
4632 def MOVIv16b_ns : SIMDModifiedImmVectorNoShift<1, 0, 0, 0b1110, V128, imm0_255,
4634 [(set (v16i8 V128:$Rd), (AArch64movi imm0_255:$imm8))]>;
4638 // EDIT per word & halfword: 2s, 4h, 4s, & 8h
4639 defm MVNI : SIMDModifiedImmVectorShift<1, 0b10, 0b00, "mvni">;
4641 def : InstAlias<"mvni $Vd.4h, $imm", (MVNIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
4642 def : InstAlias<"mvni $Vd.8h, $imm", (MVNIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
4643 def : InstAlias<"mvni $Vd.2s, $imm", (MVNIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
4644 def : InstAlias<"mvni $Vd.4s, $imm", (MVNIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
4646 def : InstAlias<"mvni.4h $Vd, $imm", (MVNIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
4647 def : InstAlias<"mvni.8h $Vd, $imm", (MVNIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
4648 def : InstAlias<"mvni.2s $Vd, $imm", (MVNIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
4649 def : InstAlias<"mvni.4s $Vd, $imm", (MVNIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
4651 def : Pat<(v2i32 (AArch64mvni_shift imm0_255:$imm8, (i32 imm:$shift))),
4652 (MVNIv2i32 imm0_255:$imm8, imm:$shift)>;
4653 def : Pat<(v4i32 (AArch64mvni_shift imm0_255:$imm8, (i32 imm:$shift))),
4654 (MVNIv4i32 imm0_255:$imm8, imm:$shift)>;
4655 def : Pat<(v4i16 (AArch64mvni_shift imm0_255:$imm8, (i32 imm:$shift))),
4656 (MVNIv4i16 imm0_255:$imm8, imm:$shift)>;
4657 def : Pat<(v8i16 (AArch64mvni_shift imm0_255:$imm8, (i32 imm:$shift))),
4658 (MVNIv8i16 imm0_255:$imm8, imm:$shift)>;
4660 // EDIT per word: 2s & 4s with MSL shifter
4661 def MVNIv2s_msl : SIMDModifiedImmMoveMSL<0, 1, {1,1,0,?}, V64, "mvni", ".2s",
4662 [(set (v2i32 V64:$Rd),
4663 (AArch64mvni_msl imm0_255:$imm8, (i32 imm:$shift)))]>;
4664 def MVNIv4s_msl : SIMDModifiedImmMoveMSL<1, 1, {1,1,0,?}, V128, "mvni", ".4s",
4665 [(set (v4i32 V128:$Rd),
4666 (AArch64mvni_msl imm0_255:$imm8, (i32 imm:$shift)))]>;
4668 //----------------------------------------------------------------------------
4669 // AdvSIMD indexed element
4670 //----------------------------------------------------------------------------
4672 let hasSideEffects = 0 in {
4673 defm FMLA : SIMDFPIndexedTied<0, 0b0001, "fmla">;
4674 defm FMLS : SIMDFPIndexedTied<0, 0b0101, "fmls">;
4677 // NOTE: Operands are reordered in the FMLA/FMLS PatFrags because the
4678 // instruction expects the addend first, while the intrinsic expects it last.
4680 // On the other hand, there are quite a few valid combinatorial options due to
4681 // the commutativity of multiplication and the fact that (-x) * y = x * (-y).
4682 defm : SIMDFPIndexedTiedPatterns<"FMLA",
4683 TriOpFrag<(fma node:$RHS, node:$MHS, node:$LHS)>>;
4684 defm : SIMDFPIndexedTiedPatterns<"FMLA",
4685 TriOpFrag<(fma node:$MHS, node:$RHS, node:$LHS)>>;
4687 defm : SIMDFPIndexedTiedPatterns<"FMLS",
4688 TriOpFrag<(fma node:$MHS, (fneg node:$RHS), node:$LHS)> >;
4689 defm : SIMDFPIndexedTiedPatterns<"FMLS",
4690 TriOpFrag<(fma node:$RHS, (fneg node:$MHS), node:$LHS)> >;
4691 defm : SIMDFPIndexedTiedPatterns<"FMLS",
4692 TriOpFrag<(fma (fneg node:$RHS), node:$MHS, node:$LHS)> >;
4693 defm : SIMDFPIndexedTiedPatterns<"FMLS",
4694 TriOpFrag<(fma (fneg node:$MHS), node:$RHS, node:$LHS)> >;
4696 multiclass FMLSIndexedAfterNegPatterns<SDPatternOperator OpNode> {
4697 // 3 variants for the .2s version: DUPLANE from 128-bit, DUPLANE from 64-bit
4699 def : Pat<(v2f32 (OpNode (v2f32 V64:$Rd), (v2f32 V64:$Rn),
4700 (AArch64duplane32 (v4f32 (fneg V128:$Rm)),
4701 VectorIndexS:$idx))),
4702 (FMLSv2i32_indexed V64:$Rd, V64:$Rn, V128:$Rm, VectorIndexS:$idx)>;
4703 def : Pat<(v2f32 (OpNode (v2f32 V64:$Rd), (v2f32 V64:$Rn),
4704 (v2f32 (AArch64duplane32
4705 (v4f32 (insert_subvector undef,
4706 (v2f32 (fneg V64:$Rm)),
4708 VectorIndexS:$idx)))),
4709 (FMLSv2i32_indexed V64:$Rd, V64:$Rn,
4710 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub),
4711 VectorIndexS:$idx)>;
4712 def : Pat<(v2f32 (OpNode (v2f32 V64:$Rd), (v2f32 V64:$Rn),
4713 (AArch64dup (f32 (fneg FPR32Op:$Rm))))),
4714 (FMLSv2i32_indexed V64:$Rd, V64:$Rn,
4715 (SUBREG_TO_REG (i32 0), FPR32Op:$Rm, ssub), (i64 0))>;
4717 // 3 variants for the .4s version: DUPLANE from 128-bit, DUPLANE from 64-bit
4719 def : Pat<(v4f32 (OpNode (v4f32 V128:$Rd), (v4f32 V128:$Rn),
4720 (AArch64duplane32 (v4f32 (fneg V128:$Rm)),
4721 VectorIndexS:$idx))),
4722 (FMLSv4i32_indexed V128:$Rd, V128:$Rn, V128:$Rm,
4723 VectorIndexS:$idx)>;
4724 def : Pat<(v4f32 (OpNode (v4f32 V128:$Rd), (v4f32 V128:$Rn),
4725 (v4f32 (AArch64duplane32
4726 (v4f32 (insert_subvector undef,
4727 (v2f32 (fneg V64:$Rm)),
4729 VectorIndexS:$idx)))),
4730 (FMLSv4i32_indexed V128:$Rd, V128:$Rn,
4731 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub),
4732 VectorIndexS:$idx)>;
4733 def : Pat<(v4f32 (OpNode (v4f32 V128:$Rd), (v4f32 V128:$Rn),
4734 (AArch64dup (f32 (fneg FPR32Op:$Rm))))),
4735 (FMLSv4i32_indexed V128:$Rd, V128:$Rn,
4736 (SUBREG_TO_REG (i32 0), FPR32Op:$Rm, ssub), (i64 0))>;
4738 // 2 variants for the .2d version: DUPLANE from 128-bit, and DUP scalar
4739 // (DUPLANE from 64-bit would be trivial).
4740 def : Pat<(v2f64 (OpNode (v2f64 V128:$Rd), (v2f64 V128:$Rn),
4741 (AArch64duplane64 (v2f64 (fneg V128:$Rm)),
4742 VectorIndexD:$idx))),
4744 V128:$Rd, V128:$Rn, V128:$Rm, VectorIndexS:$idx)>;
4745 def : Pat<(v2f64 (OpNode (v2f64 V128:$Rd), (v2f64 V128:$Rn),
4746 (AArch64dup (f64 (fneg FPR64Op:$Rm))))),
4747 (FMLSv2i64_indexed V128:$Rd, V128:$Rn,
4748 (SUBREG_TO_REG (i32 0), FPR64Op:$Rm, dsub), (i64 0))>;
4750 // 2 variants for 32-bit scalar version: extract from .2s or from .4s
4751 def : Pat<(f32 (OpNode (f32 FPR32:$Rd), (f32 FPR32:$Rn),
4752 (vector_extract (v4f32 (fneg V128:$Rm)),
4753 VectorIndexS:$idx))),
4754 (FMLSv1i32_indexed FPR32:$Rd, FPR32:$Rn,
4755 V128:$Rm, VectorIndexS:$idx)>;
4756 def : Pat<(f32 (OpNode (f32 FPR32:$Rd), (f32 FPR32:$Rn),
4757 (vector_extract (v4f32 (insert_subvector undef,
4758 (v2f32 (fneg V64:$Rm)),
4760 VectorIndexS:$idx))),
4761 (FMLSv1i32_indexed FPR32:$Rd, FPR32:$Rn,
4762 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub), VectorIndexS:$idx)>;
4764 // 1 variant for 64-bit scalar version: extract from .1d or from .2d
4765 def : Pat<(f64 (OpNode (f64 FPR64:$Rd), (f64 FPR64:$Rn),
4766 (vector_extract (v2f64 (fneg V128:$Rm)),
4767 VectorIndexS:$idx))),
4768 (FMLSv1i64_indexed FPR64:$Rd, FPR64:$Rn,
4769 V128:$Rm, VectorIndexS:$idx)>;
4772 defm : FMLSIndexedAfterNegPatterns<
4773 TriOpFrag<(fma node:$RHS, node:$MHS, node:$LHS)> >;
4774 defm : FMLSIndexedAfterNegPatterns<
4775 TriOpFrag<(fma node:$MHS, node:$RHS, node:$LHS)> >;
4777 defm FMULX : SIMDFPIndexed<1, 0b1001, "fmulx", int_aarch64_neon_fmulx>;
4778 defm FMUL : SIMDFPIndexed<0, 0b1001, "fmul", fmul>;
4780 def : Pat<(v2f32 (fmul V64:$Rn, (AArch64dup (f32 FPR32:$Rm)))),
4781 (FMULv2i32_indexed V64:$Rn,
4782 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR32:$Rm, ssub),
4784 def : Pat<(v4f32 (fmul V128:$Rn, (AArch64dup (f32 FPR32:$Rm)))),
4785 (FMULv4i32_indexed V128:$Rn,
4786 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR32:$Rm, ssub),
4788 def : Pat<(v2f64 (fmul V128:$Rn, (AArch64dup (f64 FPR64:$Rm)))),
4789 (FMULv2i64_indexed V128:$Rn,
4790 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR64:$Rm, dsub),
4793 defm SQDMULH : SIMDIndexedHS<0, 0b1100, "sqdmulh", int_aarch64_neon_sqdmulh>;
4794 defm SQRDMULH : SIMDIndexedHS<0, 0b1101, "sqrdmulh", int_aarch64_neon_sqrdmulh>;
4795 defm MLA : SIMDVectorIndexedHSTied<1, 0b0000, "mla",
4796 TriOpFrag<(add node:$LHS, (mul node:$MHS, node:$RHS))>>;
4797 defm MLS : SIMDVectorIndexedHSTied<1, 0b0100, "mls",
4798 TriOpFrag<(sub node:$LHS, (mul node:$MHS, node:$RHS))>>;
4799 defm MUL : SIMDVectorIndexedHS<0, 0b1000, "mul", mul>;
4800 defm SMLAL : SIMDVectorIndexedLongSDTied<0, 0b0010, "smlal",
4801 TriOpFrag<(add node:$LHS, (int_aarch64_neon_smull node:$MHS, node:$RHS))>>;
4802 defm SMLSL : SIMDVectorIndexedLongSDTied<0, 0b0110, "smlsl",
4803 TriOpFrag<(sub node:$LHS, (int_aarch64_neon_smull node:$MHS, node:$RHS))>>;
4804 defm SMULL : SIMDVectorIndexedLongSD<0, 0b1010, "smull",
4805 int_aarch64_neon_smull>;
4806 defm SQDMLAL : SIMDIndexedLongSQDMLXSDTied<0, 0b0011, "sqdmlal",
4807 int_aarch64_neon_sqadd>;
4808 defm SQDMLSL : SIMDIndexedLongSQDMLXSDTied<0, 0b0111, "sqdmlsl",
4809 int_aarch64_neon_sqsub>;
4810 defm SQRDMLAH : SIMDIndexedSQRDMLxHSDTied<1, 0b1101, "sqrdmlah",
4811 int_aarch64_neon_sqadd>;
4812 defm SQRDMLSH : SIMDIndexedSQRDMLxHSDTied<1, 0b1111, "sqrdmlsh",
4813 int_aarch64_neon_sqsub>;
4814 defm SQDMULL : SIMDIndexedLongSD<0, 0b1011, "sqdmull", int_aarch64_neon_sqdmull>;
4815 defm UMLAL : SIMDVectorIndexedLongSDTied<1, 0b0010, "umlal",
4816 TriOpFrag<(add node:$LHS, (int_aarch64_neon_umull node:$MHS, node:$RHS))>>;
4817 defm UMLSL : SIMDVectorIndexedLongSDTied<1, 0b0110, "umlsl",
4818 TriOpFrag<(sub node:$LHS, (int_aarch64_neon_umull node:$MHS, node:$RHS))>>;
4819 defm UMULL : SIMDVectorIndexedLongSD<1, 0b1010, "umull",
4820 int_aarch64_neon_umull>;
4822 // A scalar sqdmull with the second operand being a vector lane can be
4823 // handled directly with the indexed instruction encoding.
4824 def : Pat<(int_aarch64_neon_sqdmulls_scalar (i32 FPR32:$Rn),
4825 (vector_extract (v4i32 V128:$Vm),
4826 VectorIndexS:$idx)),
4827 (SQDMULLv1i64_indexed FPR32:$Rn, V128:$Vm, VectorIndexS:$idx)>;
4829 //----------------------------------------------------------------------------
4830 // AdvSIMD scalar shift instructions
4831 //----------------------------------------------------------------------------
4832 defm FCVTZS : SIMDFPScalarRShift<0, 0b11111, "fcvtzs">;
4833 defm FCVTZU : SIMDFPScalarRShift<1, 0b11111, "fcvtzu">;
4834 defm SCVTF : SIMDFPScalarRShift<0, 0b11100, "scvtf">;
4835 defm UCVTF : SIMDFPScalarRShift<1, 0b11100, "ucvtf">;
4836 // Codegen patterns for the above. We don't put these directly on the
4837 // instructions because TableGen's type inference can't handle the truth.
4838 // Having the same base pattern for fp <--> int totally freaks it out.
4839 def : Pat<(int_aarch64_neon_vcvtfp2fxs FPR32:$Rn, vecshiftR32:$imm),
4840 (FCVTZSs FPR32:$Rn, vecshiftR32:$imm)>;
4841 def : Pat<(int_aarch64_neon_vcvtfp2fxu FPR32:$Rn, vecshiftR32:$imm),
4842 (FCVTZUs FPR32:$Rn, vecshiftR32:$imm)>;
4843 def : Pat<(i64 (int_aarch64_neon_vcvtfp2fxs (f64 FPR64:$Rn), vecshiftR64:$imm)),
4844 (FCVTZSd FPR64:$Rn, vecshiftR64:$imm)>;
4845 def : Pat<(i64 (int_aarch64_neon_vcvtfp2fxu (f64 FPR64:$Rn), vecshiftR64:$imm)),
4846 (FCVTZUd FPR64:$Rn, vecshiftR64:$imm)>;
4847 def : Pat<(v1i64 (int_aarch64_neon_vcvtfp2fxs (v1f64 FPR64:$Rn),
4849 (FCVTZSd FPR64:$Rn, vecshiftR64:$imm)>;
4850 def : Pat<(v1i64 (int_aarch64_neon_vcvtfp2fxu (v1f64 FPR64:$Rn),
4852 (FCVTZUd FPR64:$Rn, vecshiftR64:$imm)>;
4853 def : Pat<(int_aarch64_neon_vcvtfxs2fp FPR32:$Rn, vecshiftR32:$imm),
4854 (SCVTFs FPR32:$Rn, vecshiftR32:$imm)>;
4855 def : Pat<(int_aarch64_neon_vcvtfxu2fp FPR32:$Rn, vecshiftR32:$imm),
4856 (UCVTFs FPR32:$Rn, vecshiftR32:$imm)>;
4857 def : Pat<(f64 (int_aarch64_neon_vcvtfxs2fp (i64 FPR64:$Rn), vecshiftR64:$imm)),
4858 (SCVTFd FPR64:$Rn, vecshiftR64:$imm)>;
4859 def : Pat<(f64 (int_aarch64_neon_vcvtfxu2fp (i64 FPR64:$Rn), vecshiftR64:$imm)),
4860 (UCVTFd FPR64:$Rn, vecshiftR64:$imm)>;
4861 def : Pat<(v1f64 (int_aarch64_neon_vcvtfxs2fp (v1i64 FPR64:$Rn),
4863 (SCVTFd FPR64:$Rn, vecshiftR64:$imm)>;
4864 def : Pat<(v1f64 (int_aarch64_neon_vcvtfxu2fp (v1i64 FPR64:$Rn),
4866 (UCVTFd FPR64:$Rn, vecshiftR64:$imm)>;
4868 defm SHL : SIMDScalarLShiftD< 0, 0b01010, "shl", AArch64vshl>;
4869 defm SLI : SIMDScalarLShiftDTied<1, 0b01010, "sli">;
4870 defm SQRSHRN : SIMDScalarRShiftBHS< 0, 0b10011, "sqrshrn",
4871 int_aarch64_neon_sqrshrn>;
4872 defm SQRSHRUN : SIMDScalarRShiftBHS< 1, 0b10001, "sqrshrun",
4873 int_aarch64_neon_sqrshrun>;
4874 defm SQSHLU : SIMDScalarLShiftBHSD<1, 0b01100, "sqshlu", AArch64sqshlui>;
4875 defm SQSHL : SIMDScalarLShiftBHSD<0, 0b01110, "sqshl", AArch64sqshli>;
4876 defm SQSHRN : SIMDScalarRShiftBHS< 0, 0b10010, "sqshrn",
4877 int_aarch64_neon_sqshrn>;
4878 defm SQSHRUN : SIMDScalarRShiftBHS< 1, 0b10000, "sqshrun",
4879 int_aarch64_neon_sqshrun>;
4880 defm SRI : SIMDScalarRShiftDTied< 1, 0b01000, "sri">;
4881 defm SRSHR : SIMDScalarRShiftD< 0, 0b00100, "srshr", AArch64srshri>;
4882 defm SRSRA : SIMDScalarRShiftDTied< 0, 0b00110, "srsra",
4883 TriOpFrag<(add node:$LHS,
4884 (AArch64srshri node:$MHS, node:$RHS))>>;
4885 defm SSHR : SIMDScalarRShiftD< 0, 0b00000, "sshr", AArch64vashr>;
4886 defm SSRA : SIMDScalarRShiftDTied< 0, 0b00010, "ssra",
4887 TriOpFrag<(add node:$LHS,
4888 (AArch64vashr node:$MHS, node:$RHS))>>;
4889 defm UQRSHRN : SIMDScalarRShiftBHS< 1, 0b10011, "uqrshrn",
4890 int_aarch64_neon_uqrshrn>;
4891 defm UQSHL : SIMDScalarLShiftBHSD<1, 0b01110, "uqshl", AArch64uqshli>;
4892 defm UQSHRN : SIMDScalarRShiftBHS< 1, 0b10010, "uqshrn",
4893 int_aarch64_neon_uqshrn>;
4894 defm URSHR : SIMDScalarRShiftD< 1, 0b00100, "urshr", AArch64urshri>;
4895 defm URSRA : SIMDScalarRShiftDTied< 1, 0b00110, "ursra",
4896 TriOpFrag<(add node:$LHS,
4897 (AArch64urshri node:$MHS, node:$RHS))>>;
4898 defm USHR : SIMDScalarRShiftD< 1, 0b00000, "ushr", AArch64vlshr>;
4899 defm USRA : SIMDScalarRShiftDTied< 1, 0b00010, "usra",
4900 TriOpFrag<(add node:$LHS,
4901 (AArch64vlshr node:$MHS, node:$RHS))>>;
4903 //----------------------------------------------------------------------------
4904 // AdvSIMD vector shift instructions
4905 //----------------------------------------------------------------------------
4906 defm FCVTZS:SIMDVectorRShiftSD<0, 0b11111, "fcvtzs", int_aarch64_neon_vcvtfp2fxs>;
4907 defm FCVTZU:SIMDVectorRShiftSD<1, 0b11111, "fcvtzu", int_aarch64_neon_vcvtfp2fxu>;
4908 defm SCVTF: SIMDVectorRShiftToFP<0, 0b11100, "scvtf",
4909 int_aarch64_neon_vcvtfxs2fp>;
4910 defm RSHRN : SIMDVectorRShiftNarrowBHS<0, 0b10001, "rshrn",
4911 int_aarch64_neon_rshrn>;
4912 defm SHL : SIMDVectorLShiftBHSD<0, 0b01010, "shl", AArch64vshl>;
4913 defm SHRN : SIMDVectorRShiftNarrowBHS<0, 0b10000, "shrn",
4914 BinOpFrag<(trunc (AArch64vashr node:$LHS, node:$RHS))>>;
4915 defm SLI : SIMDVectorLShiftBHSDTied<1, 0b01010, "sli", int_aarch64_neon_vsli>;
4916 def : Pat<(v1i64 (int_aarch64_neon_vsli (v1i64 FPR64:$Rd), (v1i64 FPR64:$Rn),
4917 (i32 vecshiftL64:$imm))),
4918 (SLId FPR64:$Rd, FPR64:$Rn, vecshiftL64:$imm)>;
4919 defm SQRSHRN : SIMDVectorRShiftNarrowBHS<0, 0b10011, "sqrshrn",
4920 int_aarch64_neon_sqrshrn>;
4921 defm SQRSHRUN: SIMDVectorRShiftNarrowBHS<1, 0b10001, "sqrshrun",
4922 int_aarch64_neon_sqrshrun>;
4923 defm SQSHLU : SIMDVectorLShiftBHSD<1, 0b01100, "sqshlu", AArch64sqshlui>;
4924 defm SQSHL : SIMDVectorLShiftBHSD<0, 0b01110, "sqshl", AArch64sqshli>;
4925 defm SQSHRN : SIMDVectorRShiftNarrowBHS<0, 0b10010, "sqshrn",
4926 int_aarch64_neon_sqshrn>;
4927 defm SQSHRUN : SIMDVectorRShiftNarrowBHS<1, 0b10000, "sqshrun",
4928 int_aarch64_neon_sqshrun>;
4929 defm SRI : SIMDVectorRShiftBHSDTied<1, 0b01000, "sri", int_aarch64_neon_vsri>;
4930 def : Pat<(v1i64 (int_aarch64_neon_vsri (v1i64 FPR64:$Rd), (v1i64 FPR64:$Rn),
4931 (i32 vecshiftR64:$imm))),
4932 (SRId FPR64:$Rd, FPR64:$Rn, vecshiftR64:$imm)>;
4933 defm SRSHR : SIMDVectorRShiftBHSD<0, 0b00100, "srshr", AArch64srshri>;
4934 defm SRSRA : SIMDVectorRShiftBHSDTied<0, 0b00110, "srsra",
4935 TriOpFrag<(add node:$LHS,
4936 (AArch64srshri node:$MHS, node:$RHS))> >;
4937 defm SSHLL : SIMDVectorLShiftLongBHSD<0, 0b10100, "sshll",
4938 BinOpFrag<(AArch64vshl (sext node:$LHS), node:$RHS)>>;
4940 defm SSHR : SIMDVectorRShiftBHSD<0, 0b00000, "sshr", AArch64vashr>;
4941 defm SSRA : SIMDVectorRShiftBHSDTied<0, 0b00010, "ssra",
4942 TriOpFrag<(add node:$LHS, (AArch64vashr node:$MHS, node:$RHS))>>;
4943 defm UCVTF : SIMDVectorRShiftToFP<1, 0b11100, "ucvtf",
4944 int_aarch64_neon_vcvtfxu2fp>;
4945 defm UQRSHRN : SIMDVectorRShiftNarrowBHS<1, 0b10011, "uqrshrn",
4946 int_aarch64_neon_uqrshrn>;
4947 defm UQSHL : SIMDVectorLShiftBHSD<1, 0b01110, "uqshl", AArch64uqshli>;
4948 defm UQSHRN : SIMDVectorRShiftNarrowBHS<1, 0b10010, "uqshrn",
4949 int_aarch64_neon_uqshrn>;
4950 defm URSHR : SIMDVectorRShiftBHSD<1, 0b00100, "urshr", AArch64urshri>;
4951 defm URSRA : SIMDVectorRShiftBHSDTied<1, 0b00110, "ursra",
4952 TriOpFrag<(add node:$LHS,
4953 (AArch64urshri node:$MHS, node:$RHS))> >;
4954 defm USHLL : SIMDVectorLShiftLongBHSD<1, 0b10100, "ushll",
4955 BinOpFrag<(AArch64vshl (zext node:$LHS), node:$RHS)>>;
4956 defm USHR : SIMDVectorRShiftBHSD<1, 0b00000, "ushr", AArch64vlshr>;
4957 defm USRA : SIMDVectorRShiftBHSDTied<1, 0b00010, "usra",
4958 TriOpFrag<(add node:$LHS, (AArch64vlshr node:$MHS, node:$RHS))> >;
4960 // SHRN patterns for when a logical right shift was used instead of arithmetic
4961 // (the immediate guarantees no sign bits actually end up in the result so it
4963 def : Pat<(v8i8 (trunc (AArch64vlshr (v8i16 V128:$Rn), vecshiftR16Narrow:$imm))),
4964 (SHRNv8i8_shift V128:$Rn, vecshiftR16Narrow:$imm)>;
4965 def : Pat<(v4i16 (trunc (AArch64vlshr (v4i32 V128:$Rn), vecshiftR32Narrow:$imm))),
4966 (SHRNv4i16_shift V128:$Rn, vecshiftR32Narrow:$imm)>;
4967 def : Pat<(v2i32 (trunc (AArch64vlshr (v2i64 V128:$Rn), vecshiftR64Narrow:$imm))),
4968 (SHRNv2i32_shift V128:$Rn, vecshiftR64Narrow:$imm)>;
4970 def : Pat<(v16i8 (concat_vectors (v8i8 V64:$Rd),
4971 (trunc (AArch64vlshr (v8i16 V128:$Rn),
4972 vecshiftR16Narrow:$imm)))),
4973 (SHRNv16i8_shift (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub),
4974 V128:$Rn, vecshiftR16Narrow:$imm)>;
4975 def : Pat<(v8i16 (concat_vectors (v4i16 V64:$Rd),
4976 (trunc (AArch64vlshr (v4i32 V128:$Rn),
4977 vecshiftR32Narrow:$imm)))),
4978 (SHRNv8i16_shift (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub),
4979 V128:$Rn, vecshiftR32Narrow:$imm)>;
4980 def : Pat<(v4i32 (concat_vectors (v2i32 V64:$Rd),
4981 (trunc (AArch64vlshr (v2i64 V128:$Rn),
4982 vecshiftR64Narrow:$imm)))),
4983 (SHRNv4i32_shift (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub),
4984 V128:$Rn, vecshiftR32Narrow:$imm)>;
4986 // Vector sign and zero extensions are implemented with SSHLL and USSHLL.
4987 // Anyexts are implemented as zexts.
4988 def : Pat<(v8i16 (sext (v8i8 V64:$Rn))), (SSHLLv8i8_shift V64:$Rn, (i32 0))>;
4989 def : Pat<(v8i16 (zext (v8i8 V64:$Rn))), (USHLLv8i8_shift V64:$Rn, (i32 0))>;
4990 def : Pat<(v8i16 (anyext (v8i8 V64:$Rn))), (USHLLv8i8_shift V64:$Rn, (i32 0))>;
4991 def : Pat<(v4i32 (sext (v4i16 V64:$Rn))), (SSHLLv4i16_shift V64:$Rn, (i32 0))>;
4992 def : Pat<(v4i32 (zext (v4i16 V64:$Rn))), (USHLLv4i16_shift V64:$Rn, (i32 0))>;
4993 def : Pat<(v4i32 (anyext (v4i16 V64:$Rn))), (USHLLv4i16_shift V64:$Rn, (i32 0))>;
4994 def : Pat<(v2i64 (sext (v2i32 V64:$Rn))), (SSHLLv2i32_shift V64:$Rn, (i32 0))>;
4995 def : Pat<(v2i64 (zext (v2i32 V64:$Rn))), (USHLLv2i32_shift V64:$Rn, (i32 0))>;
4996 def : Pat<(v2i64 (anyext (v2i32 V64:$Rn))), (USHLLv2i32_shift V64:$Rn, (i32 0))>;
4997 // Also match an extend from the upper half of a 128 bit source register.
4998 def : Pat<(v8i16 (anyext (v8i8 (extract_subvector V128:$Rn, (i64 8)) ))),
4999 (USHLLv16i8_shift V128:$Rn, (i32 0))>;
5000 def : Pat<(v8i16 (zext (v8i8 (extract_subvector V128:$Rn, (i64 8)) ))),
5001 (USHLLv16i8_shift V128:$Rn, (i32 0))>;
5002 def : Pat<(v8i16 (sext (v8i8 (extract_subvector V128:$Rn, (i64 8)) ))),
5003 (SSHLLv16i8_shift V128:$Rn, (i32 0))>;
5004 def : Pat<(v4i32 (anyext (v4i16 (extract_subvector V128:$Rn, (i64 4)) ))),
5005 (USHLLv8i16_shift V128:$Rn, (i32 0))>;
5006 def : Pat<(v4i32 (zext (v4i16 (extract_subvector V128:$Rn, (i64 4)) ))),
5007 (USHLLv8i16_shift V128:$Rn, (i32 0))>;
5008 def : Pat<(v4i32 (sext (v4i16 (extract_subvector V128:$Rn, (i64 4)) ))),
5009 (SSHLLv8i16_shift V128:$Rn, (i32 0))>;
5010 def : Pat<(v2i64 (anyext (v2i32 (extract_subvector V128:$Rn, (i64 2)) ))),
5011 (USHLLv4i32_shift V128:$Rn, (i32 0))>;
5012 def : Pat<(v2i64 (zext (v2i32 (extract_subvector V128:$Rn, (i64 2)) ))),
5013 (USHLLv4i32_shift V128:$Rn, (i32 0))>;
5014 def : Pat<(v2i64 (sext (v2i32 (extract_subvector V128:$Rn, (i64 2)) ))),
5015 (SSHLLv4i32_shift V128:$Rn, (i32 0))>;
5017 // Vector shift sxtl aliases
5018 def : InstAlias<"sxtl.8h $dst, $src1",
5019 (SSHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
5020 def : InstAlias<"sxtl $dst.8h, $src1.8b",
5021 (SSHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
5022 def : InstAlias<"sxtl.4s $dst, $src1",
5023 (SSHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
5024 def : InstAlias<"sxtl $dst.4s, $src1.4h",
5025 (SSHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
5026 def : InstAlias<"sxtl.2d $dst, $src1",
5027 (SSHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
5028 def : InstAlias<"sxtl $dst.2d, $src1.2s",
5029 (SSHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
5031 // Vector shift sxtl2 aliases
5032 def : InstAlias<"sxtl2.8h $dst, $src1",
5033 (SSHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
5034 def : InstAlias<"sxtl2 $dst.8h, $src1.16b",
5035 (SSHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
5036 def : InstAlias<"sxtl2.4s $dst, $src1",
5037 (SSHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
5038 def : InstAlias<"sxtl2 $dst.4s, $src1.8h",
5039 (SSHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
5040 def : InstAlias<"sxtl2.2d $dst, $src1",
5041 (SSHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
5042 def : InstAlias<"sxtl2 $dst.2d, $src1.4s",
5043 (SSHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
5045 // Vector shift uxtl aliases
5046 def : InstAlias<"uxtl.8h $dst, $src1",
5047 (USHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
5048 def : InstAlias<"uxtl $dst.8h, $src1.8b",
5049 (USHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
5050 def : InstAlias<"uxtl.4s $dst, $src1",
5051 (USHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
5052 def : InstAlias<"uxtl $dst.4s, $src1.4h",
5053 (USHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
5054 def : InstAlias<"uxtl.2d $dst, $src1",
5055 (USHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
5056 def : InstAlias<"uxtl $dst.2d, $src1.2s",
5057 (USHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
5059 // Vector shift uxtl2 aliases
5060 def : InstAlias<"uxtl2.8h $dst, $src1",
5061 (USHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
5062 def : InstAlias<"uxtl2 $dst.8h, $src1.16b",
5063 (USHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
5064 def : InstAlias<"uxtl2.4s $dst, $src1",
5065 (USHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
5066 def : InstAlias<"uxtl2 $dst.4s, $src1.8h",
5067 (USHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
5068 def : InstAlias<"uxtl2.2d $dst, $src1",
5069 (USHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
5070 def : InstAlias<"uxtl2 $dst.2d, $src1.4s",
5071 (USHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
5073 // If an integer is about to be converted to a floating point value,
5074 // just load it on the floating point unit.
5075 // These patterns are more complex because floating point loads do not
5076 // support sign extension.
5077 // The sign extension has to be explicitly added and is only supported for
5078 // one step: byte-to-half, half-to-word, word-to-doubleword.
5079 // SCVTF GPR -> FPR is 9 cycles.
5080 // SCVTF FPR -> FPR is 4 cyclces.
5081 // (sign extension with lengthen) SXTL FPR -> FPR is 2 cycles.
5082 // Therefore, we can do 2 sign extensions and one SCVTF FPR -> FPR
5083 // and still being faster.
5084 // However, this is not good for code size.
5085 // 8-bits -> float. 2 sizes step-up.
5086 class SExtLoadi8CVTf32Pat<dag addrmode, dag INST>
5087 : Pat<(f32 (sint_to_fp (i32 (sextloadi8 addrmode)))),
5088 (SCVTFv1i32 (f32 (EXTRACT_SUBREG
5093 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
5100 Requires<[NotForCodeSize, UseAlternateSExtLoadCVTF32]>;
5102 def : SExtLoadi8CVTf32Pat<(ro8.Wpat GPR64sp:$Rn, GPR32:$Rm, ro8.Wext:$ext),
5103 (LDRBroW GPR64sp:$Rn, GPR32:$Rm, ro8.Wext:$ext)>;
5104 def : SExtLoadi8CVTf32Pat<(ro8.Xpat GPR64sp:$Rn, GPR64:$Rm, ro8.Xext:$ext),
5105 (LDRBroX GPR64sp:$Rn, GPR64:$Rm, ro8.Xext:$ext)>;
5106 def : SExtLoadi8CVTf32Pat<(am_indexed8 GPR64sp:$Rn, uimm12s1:$offset),
5107 (LDRBui GPR64sp:$Rn, uimm12s1:$offset)>;
5108 def : SExtLoadi8CVTf32Pat<(am_unscaled8 GPR64sp:$Rn, simm9:$offset),
5109 (LDURBi GPR64sp:$Rn, simm9:$offset)>;
5111 // 16-bits -> float. 1 size step-up.
5112 class SExtLoadi16CVTf32Pat<dag addrmode, dag INST>
5113 : Pat<(f32 (sint_to_fp (i32 (sextloadi16 addrmode)))),
5114 (SCVTFv1i32 (f32 (EXTRACT_SUBREG
5116 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
5120 ssub)))>, Requires<[NotForCodeSize]>;
5122 def : SExtLoadi16CVTf32Pat<(ro16.Wpat GPR64sp:$Rn, GPR32:$Rm, ro16.Wext:$ext),
5123 (LDRHroW GPR64sp:$Rn, GPR32:$Rm, ro16.Wext:$ext)>;
5124 def : SExtLoadi16CVTf32Pat<(ro16.Xpat GPR64sp:$Rn, GPR64:$Rm, ro16.Xext:$ext),
5125 (LDRHroX GPR64sp:$Rn, GPR64:$Rm, ro16.Xext:$ext)>;
5126 def : SExtLoadi16CVTf32Pat<(am_indexed16 GPR64sp:$Rn, uimm12s2:$offset),
5127 (LDRHui GPR64sp:$Rn, uimm12s2:$offset)>;
5128 def : SExtLoadi16CVTf32Pat<(am_unscaled16 GPR64sp:$Rn, simm9:$offset),
5129 (LDURHi GPR64sp:$Rn, simm9:$offset)>;
5131 // 32-bits to 32-bits are handled in target specific dag combine:
5132 // performIntToFpCombine.
5133 // 64-bits integer to 32-bits floating point, not possible with
5134 // SCVTF on floating point registers (both source and destination
5135 // must have the same size).
5137 // Here are the patterns for 8, 16, 32, and 64-bits to double.
5138 // 8-bits -> double. 3 size step-up: give up.
5139 // 16-bits -> double. 2 size step.
5140 class SExtLoadi16CVTf64Pat<dag addrmode, dag INST>
5141 : Pat <(f64 (sint_to_fp (i32 (sextloadi16 addrmode)))),
5142 (SCVTFv1i64 (f64 (EXTRACT_SUBREG
5147 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
5154 Requires<[NotForCodeSize, UseAlternateSExtLoadCVTF32]>;
5156 def : SExtLoadi16CVTf64Pat<(ro16.Wpat GPR64sp:$Rn, GPR32:$Rm, ro16.Wext:$ext),
5157 (LDRHroW GPR64sp:$Rn, GPR32:$Rm, ro16.Wext:$ext)>;
5158 def : SExtLoadi16CVTf64Pat<(ro16.Xpat GPR64sp:$Rn, GPR64:$Rm, ro16.Xext:$ext),
5159 (LDRHroX GPR64sp:$Rn, GPR64:$Rm, ro16.Xext:$ext)>;
5160 def : SExtLoadi16CVTf64Pat<(am_indexed16 GPR64sp:$Rn, uimm12s2:$offset),
5161 (LDRHui GPR64sp:$Rn, uimm12s2:$offset)>;
5162 def : SExtLoadi16CVTf64Pat<(am_unscaled16 GPR64sp:$Rn, simm9:$offset),
5163 (LDURHi GPR64sp:$Rn, simm9:$offset)>;
5164 // 32-bits -> double. 1 size step-up.
5165 class SExtLoadi32CVTf64Pat<dag addrmode, dag INST>
5166 : Pat <(f64 (sint_to_fp (i32 (load addrmode)))),
5167 (SCVTFv1i64 (f64 (EXTRACT_SUBREG
5169 (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
5173 dsub)))>, Requires<[NotForCodeSize]>;
5175 def : SExtLoadi32CVTf64Pat<(ro32.Wpat GPR64sp:$Rn, GPR32:$Rm, ro32.Wext:$ext),
5176 (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro32.Wext:$ext)>;
5177 def : SExtLoadi32CVTf64Pat<(ro32.Xpat GPR64sp:$Rn, GPR64:$Rm, ro32.Xext:$ext),
5178 (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro32.Xext:$ext)>;
5179 def : SExtLoadi32CVTf64Pat<(am_indexed32 GPR64sp:$Rn, uimm12s4:$offset),
5180 (LDRSui GPR64sp:$Rn, uimm12s4:$offset)>;
5181 def : SExtLoadi32CVTf64Pat<(am_unscaled32 GPR64sp:$Rn, simm9:$offset),
5182 (LDURSi GPR64sp:$Rn, simm9:$offset)>;
5184 // 64-bits -> double are handled in target specific dag combine:
5185 // performIntToFpCombine.
5188 //----------------------------------------------------------------------------
5189 // AdvSIMD Load-Store Structure
5190 //----------------------------------------------------------------------------
5191 defm LD1 : SIMDLd1Multiple<"ld1">;
5192 defm LD2 : SIMDLd2Multiple<"ld2">;
5193 defm LD3 : SIMDLd3Multiple<"ld3">;
5194 defm LD4 : SIMDLd4Multiple<"ld4">;
5196 defm ST1 : SIMDSt1Multiple<"st1">;
5197 defm ST2 : SIMDSt2Multiple<"st2">;
5198 defm ST3 : SIMDSt3Multiple<"st3">;
5199 defm ST4 : SIMDSt4Multiple<"st4">;
5201 class Ld1Pat<ValueType ty, Instruction INST>
5202 : Pat<(ty (load GPR64sp:$Rn)), (INST GPR64sp:$Rn)>;
5204 def : Ld1Pat<v16i8, LD1Onev16b>;
5205 def : Ld1Pat<v8i16, LD1Onev8h>;
5206 def : Ld1Pat<v4i32, LD1Onev4s>;
5207 def : Ld1Pat<v2i64, LD1Onev2d>;
5208 def : Ld1Pat<v8i8, LD1Onev8b>;
5209 def : Ld1Pat<v4i16, LD1Onev4h>;
5210 def : Ld1Pat<v2i32, LD1Onev2s>;
5211 def : Ld1Pat<v1i64, LD1Onev1d>;
5213 class St1Pat<ValueType ty, Instruction INST>
5214 : Pat<(store ty:$Vt, GPR64sp:$Rn),
5215 (INST ty:$Vt, GPR64sp:$Rn)>;
5217 def : St1Pat<v16i8, ST1Onev16b>;
5218 def : St1Pat<v8i16, ST1Onev8h>;
5219 def : St1Pat<v4i32, ST1Onev4s>;
5220 def : St1Pat<v2i64, ST1Onev2d>;
5221 def : St1Pat<v8i8, ST1Onev8b>;
5222 def : St1Pat<v4i16, ST1Onev4h>;
5223 def : St1Pat<v2i32, ST1Onev2s>;
5224 def : St1Pat<v1i64, ST1Onev1d>;
5230 defm LD1R : SIMDLdR<0, 0b110, 0, "ld1r", "One", 1, 2, 4, 8>;
5231 defm LD2R : SIMDLdR<1, 0b110, 0, "ld2r", "Two", 2, 4, 8, 16>;
5232 defm LD3R : SIMDLdR<0, 0b111, 0, "ld3r", "Three", 3, 6, 12, 24>;
5233 defm LD4R : SIMDLdR<1, 0b111, 0, "ld4r", "Four", 4, 8, 16, 32>;
5234 let mayLoad = 1, hasSideEffects = 0 in {
5235 defm LD1 : SIMDLdSingleBTied<0, 0b000, "ld1", VecListOneb, GPR64pi1>;
5236 defm LD1 : SIMDLdSingleHTied<0, 0b010, 0, "ld1", VecListOneh, GPR64pi2>;
5237 defm LD1 : SIMDLdSingleSTied<0, 0b100, 0b00, "ld1", VecListOnes, GPR64pi4>;
5238 defm LD1 : SIMDLdSingleDTied<0, 0b100, 0b01, "ld1", VecListOned, GPR64pi8>;
5239 defm LD2 : SIMDLdSingleBTied<1, 0b000, "ld2", VecListTwob, GPR64pi2>;
5240 defm LD2 : SIMDLdSingleHTied<1, 0b010, 0, "ld2", VecListTwoh, GPR64pi4>;
5241 defm LD2 : SIMDLdSingleSTied<1, 0b100, 0b00, "ld2", VecListTwos, GPR64pi8>;
5242 defm LD2 : SIMDLdSingleDTied<1, 0b100, 0b01, "ld2", VecListTwod, GPR64pi16>;
5243 defm LD3 : SIMDLdSingleBTied<0, 0b001, "ld3", VecListThreeb, GPR64pi3>;
5244 defm LD3 : SIMDLdSingleHTied<0, 0b011, 0, "ld3", VecListThreeh, GPR64pi6>;
5245 defm LD3 : SIMDLdSingleSTied<0, 0b101, 0b00, "ld3", VecListThrees, GPR64pi12>;
5246 defm LD3 : SIMDLdSingleDTied<0, 0b101, 0b01, "ld3", VecListThreed, GPR64pi24>;
5247 defm LD4 : SIMDLdSingleBTied<1, 0b001, "ld4", VecListFourb, GPR64pi4>;
5248 defm LD4 : SIMDLdSingleHTied<1, 0b011, 0, "ld4", VecListFourh, GPR64pi8>;
5249 defm LD4 : SIMDLdSingleSTied<1, 0b101, 0b00, "ld4", VecListFours, GPR64pi16>;
5250 defm LD4 : SIMDLdSingleDTied<1, 0b101, 0b01, "ld4", VecListFourd, GPR64pi32>;
5253 def : Pat<(v8i8 (AArch64dup (i32 (extloadi8 GPR64sp:$Rn)))),
5254 (LD1Rv8b GPR64sp:$Rn)>;
5255 def : Pat<(v16i8 (AArch64dup (i32 (extloadi8 GPR64sp:$Rn)))),
5256 (LD1Rv16b GPR64sp:$Rn)>;
5257 def : Pat<(v4i16 (AArch64dup (i32 (extloadi16 GPR64sp:$Rn)))),
5258 (LD1Rv4h GPR64sp:$Rn)>;
5259 def : Pat<(v8i16 (AArch64dup (i32 (extloadi16 GPR64sp:$Rn)))),
5260 (LD1Rv8h GPR64sp:$Rn)>;
5261 def : Pat<(v2i32 (AArch64dup (i32 (load GPR64sp:$Rn)))),
5262 (LD1Rv2s GPR64sp:$Rn)>;
5263 def : Pat<(v4i32 (AArch64dup (i32 (load GPR64sp:$Rn)))),
5264 (LD1Rv4s GPR64sp:$Rn)>;
5265 def : Pat<(v2i64 (AArch64dup (i64 (load GPR64sp:$Rn)))),
5266 (LD1Rv2d GPR64sp:$Rn)>;
5267 def : Pat<(v1i64 (AArch64dup (i64 (load GPR64sp:$Rn)))),
5268 (LD1Rv1d GPR64sp:$Rn)>;
5269 // Grab the floating point version too
5270 def : Pat<(v2f32 (AArch64dup (f32 (load GPR64sp:$Rn)))),
5271 (LD1Rv2s GPR64sp:$Rn)>;
5272 def : Pat<(v4f32 (AArch64dup (f32 (load GPR64sp:$Rn)))),
5273 (LD1Rv4s GPR64sp:$Rn)>;
5274 def : Pat<(v2f64 (AArch64dup (f64 (load GPR64sp:$Rn)))),
5275 (LD1Rv2d GPR64sp:$Rn)>;
5276 def : Pat<(v1f64 (AArch64dup (f64 (load GPR64sp:$Rn)))),
5277 (LD1Rv1d GPR64sp:$Rn)>;
5278 def : Pat<(v4f16 (AArch64dup (f16 (load GPR64sp:$Rn)))),
5279 (LD1Rv4h GPR64sp:$Rn)>;
5280 def : Pat<(v8f16 (AArch64dup (f16 (load GPR64sp:$Rn)))),
5281 (LD1Rv8h GPR64sp:$Rn)>;
5283 class Ld1Lane128Pat<SDPatternOperator scalar_load, Operand VecIndex,
5284 ValueType VTy, ValueType STy, Instruction LD1>
5285 : Pat<(vector_insert (VTy VecListOne128:$Rd),
5286 (STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
5287 (LD1 VecListOne128:$Rd, VecIndex:$idx, GPR64sp:$Rn)>;
5289 def : Ld1Lane128Pat<extloadi8, VectorIndexB, v16i8, i32, LD1i8>;
5290 def : Ld1Lane128Pat<extloadi16, VectorIndexH, v8i16, i32, LD1i16>;
5291 def : Ld1Lane128Pat<load, VectorIndexS, v4i32, i32, LD1i32>;
5292 def : Ld1Lane128Pat<load, VectorIndexS, v4f32, f32, LD1i32>;
5293 def : Ld1Lane128Pat<load, VectorIndexD, v2i64, i64, LD1i64>;
5294 def : Ld1Lane128Pat<load, VectorIndexD, v2f64, f64, LD1i64>;
5295 def : Ld1Lane128Pat<load, VectorIndexH, v8f16, f16, LD1i16>;
5297 class Ld1Lane64Pat<SDPatternOperator scalar_load, Operand VecIndex,
5298 ValueType VTy, ValueType STy, Instruction LD1>
5299 : Pat<(vector_insert (VTy VecListOne64:$Rd),
5300 (STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
5302 (LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
5303 VecIndex:$idx, GPR64sp:$Rn),
5306 def : Ld1Lane64Pat<extloadi8, VectorIndexB, v8i8, i32, LD1i8>;
5307 def : Ld1Lane64Pat<extloadi16, VectorIndexH, v4i16, i32, LD1i16>;
5308 def : Ld1Lane64Pat<load, VectorIndexS, v2i32, i32, LD1i32>;
5309 def : Ld1Lane64Pat<load, VectorIndexS, v2f32, f32, LD1i32>;
5310 def : Ld1Lane64Pat<load, VectorIndexH, v4f16, f16, LD1i16>;
5313 defm LD1 : SIMDLdSt1SingleAliases<"ld1">;
5314 defm LD2 : SIMDLdSt2SingleAliases<"ld2">;
5315 defm LD3 : SIMDLdSt3SingleAliases<"ld3">;
5316 defm LD4 : SIMDLdSt4SingleAliases<"ld4">;
5319 defm ST1 : SIMDStSingleB<0, 0b000, "st1", VecListOneb, GPR64pi1>;
5320 defm ST1 : SIMDStSingleH<0, 0b010, 0, "st1", VecListOneh, GPR64pi2>;
5321 defm ST1 : SIMDStSingleS<0, 0b100, 0b00, "st1", VecListOnes, GPR64pi4>;
5322 defm ST1 : SIMDStSingleD<0, 0b100, 0b01, "st1", VecListOned, GPR64pi8>;
5324 let AddedComplexity = 19 in
5325 class St1Lane128Pat<SDPatternOperator scalar_store, Operand VecIndex,
5326 ValueType VTy, ValueType STy, Instruction ST1>
5328 (STy (vector_extract (VTy VecListOne128:$Vt), VecIndex:$idx)),
5330 (ST1 VecListOne128:$Vt, VecIndex:$idx, GPR64sp:$Rn)>;
5332 def : St1Lane128Pat<truncstorei8, VectorIndexB, v16i8, i32, ST1i8>;
5333 def : St1Lane128Pat<truncstorei16, VectorIndexH, v8i16, i32, ST1i16>;
5334 def : St1Lane128Pat<store, VectorIndexS, v4i32, i32, ST1i32>;
5335 def : St1Lane128Pat<store, VectorIndexS, v4f32, f32, ST1i32>;
5336 def : St1Lane128Pat<store, VectorIndexD, v2i64, i64, ST1i64>;
5337 def : St1Lane128Pat<store, VectorIndexD, v2f64, f64, ST1i64>;
5338 def : St1Lane128Pat<store, VectorIndexH, v8f16, f16, ST1i16>;
5340 let AddedComplexity = 19 in
5341 class St1Lane64Pat<SDPatternOperator scalar_store, Operand VecIndex,
5342 ValueType VTy, ValueType STy, Instruction ST1>
5344 (STy (vector_extract (VTy VecListOne64:$Vt), VecIndex:$idx)),
5346 (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
5347 VecIndex:$idx, GPR64sp:$Rn)>;
5349 def : St1Lane64Pat<truncstorei8, VectorIndexB, v8i8, i32, ST1i8>;
5350 def : St1Lane64Pat<truncstorei16, VectorIndexH, v4i16, i32, ST1i16>;
5351 def : St1Lane64Pat<store, VectorIndexS, v2i32, i32, ST1i32>;
5352 def : St1Lane64Pat<store, VectorIndexS, v2f32, f32, ST1i32>;
5353 def : St1Lane64Pat<store, VectorIndexH, v4f16, f16, ST1i16>;
5355 multiclass St1LanePost64Pat<SDPatternOperator scalar_store, Operand VecIndex,
5356 ValueType VTy, ValueType STy, Instruction ST1,
5358 def : Pat<(scalar_store
5359 (STy (vector_extract (VTy VecListOne64:$Vt), VecIndex:$idx)),
5360 GPR64sp:$Rn, offset),
5361 (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
5362 VecIndex:$idx, GPR64sp:$Rn, XZR)>;
5364 def : Pat<(scalar_store
5365 (STy (vector_extract (VTy VecListOne64:$Vt), VecIndex:$idx)),
5366 GPR64sp:$Rn, GPR64:$Rm),
5367 (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
5368 VecIndex:$idx, GPR64sp:$Rn, $Rm)>;
5371 defm : St1LanePost64Pat<post_truncsti8, VectorIndexB, v8i8, i32, ST1i8_POST, 1>;
5372 defm : St1LanePost64Pat<post_truncsti16, VectorIndexH, v4i16, i32, ST1i16_POST,
5374 defm : St1LanePost64Pat<post_store, VectorIndexS, v2i32, i32, ST1i32_POST, 4>;
5375 defm : St1LanePost64Pat<post_store, VectorIndexS, v2f32, f32, ST1i32_POST, 4>;
5376 defm : St1LanePost64Pat<post_store, VectorIndexD, v1i64, i64, ST1i64_POST, 8>;
5377 defm : St1LanePost64Pat<post_store, VectorIndexD, v1f64, f64, ST1i64_POST, 8>;
5378 defm : St1LanePost64Pat<post_store, VectorIndexH, v4f16, f16, ST1i16_POST, 2>;
5380 multiclass St1LanePost128Pat<SDPatternOperator scalar_store, Operand VecIndex,
5381 ValueType VTy, ValueType STy, Instruction ST1,
5383 def : Pat<(scalar_store
5384 (STy (vector_extract (VTy VecListOne128:$Vt), VecIndex:$idx)),
5385 GPR64sp:$Rn, offset),
5386 (ST1 VecListOne128:$Vt, VecIndex:$idx, GPR64sp:$Rn, XZR)>;
5388 def : Pat<(scalar_store
5389 (STy (vector_extract (VTy VecListOne128:$Vt), VecIndex:$idx)),
5390 GPR64sp:$Rn, GPR64:$Rm),
5391 (ST1 VecListOne128:$Vt, VecIndex:$idx, GPR64sp:$Rn, $Rm)>;
5394 defm : St1LanePost128Pat<post_truncsti8, VectorIndexB, v16i8, i32, ST1i8_POST,
5396 defm : St1LanePost128Pat<post_truncsti16, VectorIndexH, v8i16, i32, ST1i16_POST,
5398 defm : St1LanePost128Pat<post_store, VectorIndexS, v4i32, i32, ST1i32_POST, 4>;
5399 defm : St1LanePost128Pat<post_store, VectorIndexS, v4f32, f32, ST1i32_POST, 4>;
5400 defm : St1LanePost128Pat<post_store, VectorIndexD, v2i64, i64, ST1i64_POST, 8>;
5401 defm : St1LanePost128Pat<post_store, VectorIndexD, v2f64, f64, ST1i64_POST, 8>;
5402 defm : St1LanePost128Pat<post_store, VectorIndexH, v8f16, f16, ST1i16_POST, 2>;
5404 let mayStore = 1, hasSideEffects = 0 in {
5405 defm ST2 : SIMDStSingleB<1, 0b000, "st2", VecListTwob, GPR64pi2>;
5406 defm ST2 : SIMDStSingleH<1, 0b010, 0, "st2", VecListTwoh, GPR64pi4>;
5407 defm ST2 : SIMDStSingleS<1, 0b100, 0b00, "st2", VecListTwos, GPR64pi8>;
5408 defm ST2 : SIMDStSingleD<1, 0b100, 0b01, "st2", VecListTwod, GPR64pi16>;
5409 defm ST3 : SIMDStSingleB<0, 0b001, "st3", VecListThreeb, GPR64pi3>;
5410 defm ST3 : SIMDStSingleH<0, 0b011, 0, "st3", VecListThreeh, GPR64pi6>;
5411 defm ST3 : SIMDStSingleS<0, 0b101, 0b00, "st3", VecListThrees, GPR64pi12>;
5412 defm ST3 : SIMDStSingleD<0, 0b101, 0b01, "st3", VecListThreed, GPR64pi24>;
5413 defm ST4 : SIMDStSingleB<1, 0b001, "st4", VecListFourb, GPR64pi4>;
5414 defm ST4 : SIMDStSingleH<1, 0b011, 0, "st4", VecListFourh, GPR64pi8>;
5415 defm ST4 : SIMDStSingleS<1, 0b101, 0b00, "st4", VecListFours, GPR64pi16>;
5416 defm ST4 : SIMDStSingleD<1, 0b101, 0b01, "st4", VecListFourd, GPR64pi32>;
5419 defm ST1 : SIMDLdSt1SingleAliases<"st1">;
5420 defm ST2 : SIMDLdSt2SingleAliases<"st2">;
5421 defm ST3 : SIMDLdSt3SingleAliases<"st3">;
5422 defm ST4 : SIMDLdSt4SingleAliases<"st4">;
5424 //----------------------------------------------------------------------------
5425 // Crypto extensions
5426 //----------------------------------------------------------------------------
5428 def AESErr : AESTiedInst<0b0100, "aese", int_aarch64_crypto_aese>;
5429 def AESDrr : AESTiedInst<0b0101, "aesd", int_aarch64_crypto_aesd>;
5430 def AESMCrr : AESInst< 0b0110, "aesmc", int_aarch64_crypto_aesmc>;
5431 def AESIMCrr : AESInst< 0b0111, "aesimc", int_aarch64_crypto_aesimc>;
5433 // Pseudo instructions for AESMCrr/AESIMCrr with a register constraint required
5434 // for AES fusion on some CPUs.
5435 let hasSideEffects = 0, mayStore = 0, mayLoad = 0 in {
5436 def AESMCrrTied: Pseudo<(outs V128:$Rd), (ins V128:$Rn), [], "$Rn = $Rd">,
5438 def AESIMCrrTied: Pseudo<(outs V128:$Rd), (ins V128:$Rn), [], "$Rn = $Rd">,
5442 // Only use constrained versions of AES(I)MC instructions if they are paired with
5444 def : Pat<(v16i8 (int_aarch64_crypto_aesmc
5445 (v16i8 (int_aarch64_crypto_aese (v16i8 V128:$src1),
5446 (v16i8 V128:$src2))))),
5447 (v16i8 (AESMCrrTied (v16i8 (AESErr (v16i8 V128:$src1),
5448 (v16i8 V128:$src2)))))>,
5449 Requires<[HasFuseAES]>;
5451 def : Pat<(v16i8 (int_aarch64_crypto_aesimc
5452 (v16i8 (int_aarch64_crypto_aesd (v16i8 V128:$src1),
5453 (v16i8 V128:$src2))))),
5454 (v16i8 (AESIMCrrTied (v16i8 (AESDrr (v16i8 V128:$src1),
5455 (v16i8 V128:$src2)))))>,
5456 Requires<[HasFuseAES]>;
5458 def SHA1Crrr : SHATiedInstQSV<0b000, "sha1c", int_aarch64_crypto_sha1c>;
5459 def SHA1Prrr : SHATiedInstQSV<0b001, "sha1p", int_aarch64_crypto_sha1p>;
5460 def SHA1Mrrr : SHATiedInstQSV<0b010, "sha1m", int_aarch64_crypto_sha1m>;
5461 def SHA1SU0rrr : SHATiedInstVVV<0b011, "sha1su0", int_aarch64_crypto_sha1su0>;
5462 def SHA256Hrrr : SHATiedInstQQV<0b100, "sha256h", int_aarch64_crypto_sha256h>;
5463 def SHA256H2rrr : SHATiedInstQQV<0b101, "sha256h2",int_aarch64_crypto_sha256h2>;
5464 def SHA256SU1rrr :SHATiedInstVVV<0b110, "sha256su1",int_aarch64_crypto_sha256su1>;
5466 def SHA1Hrr : SHAInstSS< 0b0000, "sha1h", int_aarch64_crypto_sha1h>;
5467 def SHA1SU1rr : SHATiedInstVV<0b0001, "sha1su1", int_aarch64_crypto_sha1su1>;
5468 def SHA256SU0rr : SHATiedInstVV<0b0010, "sha256su0",int_aarch64_crypto_sha256su0>;
5470 //----------------------------------------------------------------------------
5472 //----------------------------------------------------------------------------
5473 // FIXME: Like for X86, these should go in their own separate .td file.
5475 def def32 : PatLeaf<(i32 GPR32:$src), [{
5479 // In the case of a 32-bit def that is known to implicitly zero-extend,
5480 // we can use a SUBREG_TO_REG.
5481 def : Pat<(i64 (zext def32:$src)), (SUBREG_TO_REG (i64 0), GPR32:$src, sub_32)>;
5483 // For an anyext, we don't care what the high bits are, so we can perform an
5484 // INSERT_SUBREF into an IMPLICIT_DEF.
5485 def : Pat<(i64 (anyext GPR32:$src)),
5486 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>;
5488 // When we need to explicitly zero-extend, we use a 32-bit MOV instruction and
5489 // then assert the extension has happened.
5490 def : Pat<(i64 (zext GPR32:$src)),
5491 (SUBREG_TO_REG (i32 0), (ORRWrs WZR, GPR32:$src, 0), sub_32)>;
5493 // To sign extend, we use a signed bitfield move instruction (SBFM) on the
5494 // containing super-reg.
5495 def : Pat<(i64 (sext GPR32:$src)),
5496 (SBFMXri (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32), 0, 31)>;
5497 def : Pat<(i64 (sext_inreg GPR64:$src, i32)), (SBFMXri GPR64:$src, 0, 31)>;
5498 def : Pat<(i64 (sext_inreg GPR64:$src, i16)), (SBFMXri GPR64:$src, 0, 15)>;
5499 def : Pat<(i64 (sext_inreg GPR64:$src, i8)), (SBFMXri GPR64:$src, 0, 7)>;
5500 def : Pat<(i64 (sext_inreg GPR64:$src, i1)), (SBFMXri GPR64:$src, 0, 0)>;
5501 def : Pat<(i32 (sext_inreg GPR32:$src, i16)), (SBFMWri GPR32:$src, 0, 15)>;
5502 def : Pat<(i32 (sext_inreg GPR32:$src, i8)), (SBFMWri GPR32:$src, 0, 7)>;
5503 def : Pat<(i32 (sext_inreg GPR32:$src, i1)), (SBFMWri GPR32:$src, 0, 0)>;
5505 def : Pat<(shl (sext_inreg GPR32:$Rn, i8), (i64 imm0_31:$imm)),
5506 (SBFMWri GPR32:$Rn, (i64 (i32shift_a imm0_31:$imm)),
5507 (i64 (i32shift_sext_i8 imm0_31:$imm)))>;
5508 def : Pat<(shl (sext_inreg GPR64:$Rn, i8), (i64 imm0_63:$imm)),
5509 (SBFMXri GPR64:$Rn, (i64 (i64shift_a imm0_63:$imm)),
5510 (i64 (i64shift_sext_i8 imm0_63:$imm)))>;
5512 def : Pat<(shl (sext_inreg GPR32:$Rn, i16), (i64 imm0_31:$imm)),
5513 (SBFMWri GPR32:$Rn, (i64 (i32shift_a imm0_31:$imm)),
5514 (i64 (i32shift_sext_i16 imm0_31:$imm)))>;
5515 def : Pat<(shl (sext_inreg GPR64:$Rn, i16), (i64 imm0_63:$imm)),
5516 (SBFMXri GPR64:$Rn, (i64 (i64shift_a imm0_63:$imm)),
5517 (i64 (i64shift_sext_i16 imm0_63:$imm)))>;
5519 def : Pat<(shl (i64 (sext GPR32:$Rn)), (i64 imm0_63:$imm)),
5520 (SBFMXri (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$Rn, sub_32),
5521 (i64 (i64shift_a imm0_63:$imm)),
5522 (i64 (i64shift_sext_i32 imm0_63:$imm)))>;
5524 // sra patterns have an AddedComplexity of 10, so make sure we have a higher
5525 // AddedComplexity for the following patterns since we want to match sext + sra
5526 // patterns before we attempt to match a single sra node.
5527 let AddedComplexity = 20 in {
5528 // We support all sext + sra combinations which preserve at least one bit of the
5529 // original value which is to be sign extended. E.g. we support shifts up to
5531 def : Pat<(sra (sext_inreg GPR32:$Rn, i8), (i64 imm0_7:$imm)),
5532 (SBFMWri GPR32:$Rn, (i64 imm0_7:$imm), 7)>;
5533 def : Pat<(sra (sext_inreg GPR64:$Rn, i8), (i64 imm0_7:$imm)),
5534 (SBFMXri GPR64:$Rn, (i64 imm0_7:$imm), 7)>;
5536 def : Pat<(sra (sext_inreg GPR32:$Rn, i16), (i64 imm0_15:$imm)),
5537 (SBFMWri GPR32:$Rn, (i64 imm0_15:$imm), 15)>;
5538 def : Pat<(sra (sext_inreg GPR64:$Rn, i16), (i64 imm0_15:$imm)),
5539 (SBFMXri GPR64:$Rn, (i64 imm0_15:$imm), 15)>;
5541 def : Pat<(sra (i64 (sext GPR32:$Rn)), (i64 imm0_31:$imm)),
5542 (SBFMXri (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$Rn, sub_32),
5543 (i64 imm0_31:$imm), 31)>;
5544 } // AddedComplexity = 20
5546 // To truncate, we can simply extract from a subregister.
5547 def : Pat<(i32 (trunc GPR64sp:$src)),
5548 (i32 (EXTRACT_SUBREG GPR64sp:$src, sub_32))>;
5550 // __builtin_trap() uses the BRK instruction on AArch64.
5551 def : Pat<(trap), (BRK 1)>;
5553 // Conversions within AdvSIMD types in the same register size are free.
5554 // But because we need a consistent lane ordering, in big endian many
5555 // conversions require one or more REV instructions.
5557 // Consider a simple memory load followed by a bitconvert then a store.
5559 // v1 = BITCAST v2i32 v0 to v4i16
5562 // In big endian mode every memory access has an implicit byte swap. LDR and
5563 // STR do a 64-bit byte swap, whereas LD1/ST1 do a byte swap per lane - that
5564 // is, they treat the vector as a sequence of elements to be byte-swapped.
5565 // The two pairs of instructions are fundamentally incompatible. We've decided
5566 // to use LD1/ST1 only to simplify compiler implementation.
5568 // LD1/ST1 perform the equivalent of a sequence of LDR/STR + REV. This makes
5569 // the original code sequence:
5571 // v1 = REV v2i32 (implicit)
5572 // v2 = BITCAST v2i32 v1 to v4i16
5573 // v3 = REV v4i16 v2 (implicit)
5576 // But this is now broken - the value stored is different to the value loaded
5577 // due to lane reordering. To fix this, on every BITCAST we must perform two
5580 // v1 = REV v2i32 (implicit)
5582 // v3 = BITCAST v2i32 v2 to v4i16
5584 // v5 = REV v4i16 v4 (implicit)
5587 // This means an extra two instructions, but actually in most cases the two REV
5588 // instructions can be combined into one. For example:
5589 // (REV64_2s (REV64_4h X)) === (REV32_4h X)
5591 // There is also no 128-bit REV instruction. This must be synthesized with an
5594 // Most bitconverts require some sort of conversion. The only exceptions are:
5595 // a) Identity conversions - vNfX <-> vNiX
5596 // b) Single-lane-to-scalar - v1fX <-> fX or v1iX <-> iX
5599 // Natural vector casts (64 bit)
5600 def : Pat<(v8i8 (AArch64NvCast (v2i32 FPR64:$src))), (v8i8 FPR64:$src)>;
5601 def : Pat<(v4i16 (AArch64NvCast (v2i32 FPR64:$src))), (v4i16 FPR64:$src)>;
5602 def : Pat<(v4f16 (AArch64NvCast (v2i32 FPR64:$src))), (v4f16 FPR64:$src)>;
5603 def : Pat<(v2i32 (AArch64NvCast (v2i32 FPR64:$src))), (v2i32 FPR64:$src)>;
5604 def : Pat<(v2f32 (AArch64NvCast (v2i32 FPR64:$src))), (v2f32 FPR64:$src)>;
5605 def : Pat<(v1i64 (AArch64NvCast (v2i32 FPR64:$src))), (v1i64 FPR64:$src)>;
5607 def : Pat<(v8i8 (AArch64NvCast (v4i16 FPR64:$src))), (v8i8 FPR64:$src)>;
5608 def : Pat<(v4i16 (AArch64NvCast (v4i16 FPR64:$src))), (v4i16 FPR64:$src)>;
5609 def : Pat<(v4f16 (AArch64NvCast (v4i16 FPR64:$src))), (v4f16 FPR64:$src)>;
5610 def : Pat<(v2i32 (AArch64NvCast (v4i16 FPR64:$src))), (v2i32 FPR64:$src)>;
5611 def : Pat<(v1i64 (AArch64NvCast (v4i16 FPR64:$src))), (v1i64 FPR64:$src)>;
5613 def : Pat<(v8i8 (AArch64NvCast (v8i8 FPR64:$src))), (v8i8 FPR64:$src)>;
5614 def : Pat<(v4i16 (AArch64NvCast (v8i8 FPR64:$src))), (v4i16 FPR64:$src)>;
5615 def : Pat<(v4f16 (AArch64NvCast (v8i8 FPR64:$src))), (v4f16 FPR64:$src)>;
5616 def : Pat<(v2i32 (AArch64NvCast (v8i8 FPR64:$src))), (v2i32 FPR64:$src)>;
5617 def : Pat<(v1i64 (AArch64NvCast (v8i8 FPR64:$src))), (v1i64 FPR64:$src)>;
5619 def : Pat<(v8i8 (AArch64NvCast (f64 FPR64:$src))), (v8i8 FPR64:$src)>;
5620 def : Pat<(v4i16 (AArch64NvCast (f64 FPR64:$src))), (v4i16 FPR64:$src)>;
5621 def : Pat<(v4f16 (AArch64NvCast (f64 FPR64:$src))), (v4f16 FPR64:$src)>;
5622 def : Pat<(v2i32 (AArch64NvCast (f64 FPR64:$src))), (v2i32 FPR64:$src)>;
5623 def : Pat<(v2f32 (AArch64NvCast (f64 FPR64:$src))), (v2f32 FPR64:$src)>;
5624 def : Pat<(v1i64 (AArch64NvCast (f64 FPR64:$src))), (v1i64 FPR64:$src)>;
5625 def : Pat<(v1f64 (AArch64NvCast (f64 FPR64:$src))), (v1f64 FPR64:$src)>;
5627 def : Pat<(v8i8 (AArch64NvCast (v2f32 FPR64:$src))), (v8i8 FPR64:$src)>;
5628 def : Pat<(v4i16 (AArch64NvCast (v2f32 FPR64:$src))), (v4i16 FPR64:$src)>;
5629 def : Pat<(v2i32 (AArch64NvCast (v2f32 FPR64:$src))), (v2i32 FPR64:$src)>;
5630 def : Pat<(v2f32 (AArch64NvCast (v2f32 FPR64:$src))), (v2f32 FPR64:$src)>;
5631 def : Pat<(v1i64 (AArch64NvCast (v2f32 FPR64:$src))), (v1i64 FPR64:$src)>;
5633 // Natural vector casts (128 bit)
5634 def : Pat<(v16i8 (AArch64NvCast (v4i32 FPR128:$src))), (v16i8 FPR128:$src)>;
5635 def : Pat<(v8i16 (AArch64NvCast (v4i32 FPR128:$src))), (v8i16 FPR128:$src)>;
5636 def : Pat<(v8f16 (AArch64NvCast (v4i32 FPR128:$src))), (v8f16 FPR128:$src)>;
5637 def : Pat<(v4i32 (AArch64NvCast (v4i32 FPR128:$src))), (v4i32 FPR128:$src)>;
5638 def : Pat<(v4f32 (AArch64NvCast (v4i32 FPR128:$src))), (v4f32 FPR128:$src)>;
5639 def : Pat<(v2i64 (AArch64NvCast (v4i32 FPR128:$src))), (v2i64 FPR128:$src)>;
5640 def : Pat<(v2f64 (AArch64NvCast (v4i32 FPR128:$src))), (v2f64 FPR128:$src)>;
5642 def : Pat<(v16i8 (AArch64NvCast (v8i16 FPR128:$src))), (v16i8 FPR128:$src)>;
5643 def : Pat<(v8i16 (AArch64NvCast (v8i16 FPR128:$src))), (v8i16 FPR128:$src)>;
5644 def : Pat<(v8f16 (AArch64NvCast (v8i16 FPR128:$src))), (v8f16 FPR128:$src)>;
5645 def : Pat<(v4i32 (AArch64NvCast (v8i16 FPR128:$src))), (v4i32 FPR128:$src)>;
5646 def : Pat<(v2i64 (AArch64NvCast (v8i16 FPR128:$src))), (v2i64 FPR128:$src)>;
5647 def : Pat<(v4f32 (AArch64NvCast (v8i16 FPR128:$src))), (v4f32 FPR128:$src)>;
5648 def : Pat<(v2f64 (AArch64NvCast (v8i16 FPR128:$src))), (v2f64 FPR128:$src)>;
5650 def : Pat<(v16i8 (AArch64NvCast (v16i8 FPR128:$src))), (v16i8 FPR128:$src)>;
5651 def : Pat<(v8i16 (AArch64NvCast (v16i8 FPR128:$src))), (v8i16 FPR128:$src)>;
5652 def : Pat<(v8f16 (AArch64NvCast (v16i8 FPR128:$src))), (v8f16 FPR128:$src)>;
5653 def : Pat<(v4i32 (AArch64NvCast (v16i8 FPR128:$src))), (v4i32 FPR128:$src)>;
5654 def : Pat<(v2i64 (AArch64NvCast (v16i8 FPR128:$src))), (v2i64 FPR128:$src)>;
5655 def : Pat<(v4f32 (AArch64NvCast (v16i8 FPR128:$src))), (v4f32 FPR128:$src)>;
5656 def : Pat<(v2f64 (AArch64NvCast (v16i8 FPR128:$src))), (v2f64 FPR128:$src)>;
5658 def : Pat<(v16i8 (AArch64NvCast (v2i64 FPR128:$src))), (v16i8 FPR128:$src)>;
5659 def : Pat<(v8i16 (AArch64NvCast (v2i64 FPR128:$src))), (v8i16 FPR128:$src)>;
5660 def : Pat<(v8f16 (AArch64NvCast (v2i64 FPR128:$src))), (v8f16 FPR128:$src)>;
5661 def : Pat<(v4i32 (AArch64NvCast (v2i64 FPR128:$src))), (v4i32 FPR128:$src)>;
5662 def : Pat<(v2i64 (AArch64NvCast (v2i64 FPR128:$src))), (v2i64 FPR128:$src)>;
5663 def : Pat<(v4f32 (AArch64NvCast (v2i64 FPR128:$src))), (v4f32 FPR128:$src)>;
5664 def : Pat<(v2f64 (AArch64NvCast (v2i64 FPR128:$src))), (v2f64 FPR128:$src)>;
5666 def : Pat<(v16i8 (AArch64NvCast (v4f32 FPR128:$src))), (v16i8 FPR128:$src)>;
5667 def : Pat<(v8i16 (AArch64NvCast (v4f32 FPR128:$src))), (v8i16 FPR128:$src)>;
5668 def : Pat<(v4i32 (AArch64NvCast (v4f32 FPR128:$src))), (v4i32 FPR128:$src)>;
5669 def : Pat<(v4f32 (AArch64NvCast (v4f32 FPR128:$src))), (v4f32 FPR128:$src)>;
5670 def : Pat<(v2i64 (AArch64NvCast (v4f32 FPR128:$src))), (v2i64 FPR128:$src)>;
5671 def : Pat<(v8f16 (AArch64NvCast (v4f32 FPR128:$src))), (v8f16 FPR128:$src)>;
5672 def : Pat<(v2f64 (AArch64NvCast (v4f32 FPR128:$src))), (v2f64 FPR128:$src)>;
5674 def : Pat<(v16i8 (AArch64NvCast (v2f64 FPR128:$src))), (v16i8 FPR128:$src)>;
5675 def : Pat<(v8i16 (AArch64NvCast (v2f64 FPR128:$src))), (v8i16 FPR128:$src)>;
5676 def : Pat<(v4i32 (AArch64NvCast (v2f64 FPR128:$src))), (v4i32 FPR128:$src)>;
5677 def : Pat<(v2i64 (AArch64NvCast (v2f64 FPR128:$src))), (v2i64 FPR128:$src)>;
5678 def : Pat<(v2f64 (AArch64NvCast (v2f64 FPR128:$src))), (v2f64 FPR128:$src)>;
5679 def : Pat<(v8f16 (AArch64NvCast (v2f64 FPR128:$src))), (v8f16 FPR128:$src)>;
5680 def : Pat<(v4f32 (AArch64NvCast (v2f64 FPR128:$src))), (v4f32 FPR128:$src)>;
5682 let Predicates = [IsLE] in {
5683 def : Pat<(v8i8 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5684 def : Pat<(v4i16 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5685 def : Pat<(v2i32 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5686 def : Pat<(v4f16 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5687 def : Pat<(v2f32 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5689 def : Pat<(i64 (bitconvert (v8i8 V64:$Vn))),
5690 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5691 def : Pat<(i64 (bitconvert (v4i16 V64:$Vn))),
5692 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5693 def : Pat<(i64 (bitconvert (v2i32 V64:$Vn))),
5694 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5695 def : Pat<(i64 (bitconvert (v4f16 V64:$Vn))),
5696 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5697 def : Pat<(i64 (bitconvert (v2f32 V64:$Vn))),
5698 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5699 def : Pat<(i64 (bitconvert (v1f64 V64:$Vn))),
5700 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5702 let Predicates = [IsBE] in {
5703 def : Pat<(v8i8 (bitconvert GPR64:$Xn)),
5704 (REV64v8i8 (COPY_TO_REGCLASS GPR64:$Xn, FPR64))>;
5705 def : Pat<(v4i16 (bitconvert GPR64:$Xn)),
5706 (REV64v4i16 (COPY_TO_REGCLASS GPR64:$Xn, FPR64))>;
5707 def : Pat<(v2i32 (bitconvert GPR64:$Xn)),
5708 (REV64v2i32 (COPY_TO_REGCLASS GPR64:$Xn, FPR64))>;
5709 def : Pat<(v4f16 (bitconvert GPR64:$Xn)),
5710 (REV64v4i16 (COPY_TO_REGCLASS GPR64:$Xn, FPR64))>;
5711 def : Pat<(v2f32 (bitconvert GPR64:$Xn)),
5712 (REV64v2i32 (COPY_TO_REGCLASS GPR64:$Xn, FPR64))>;
5714 def : Pat<(i64 (bitconvert (v8i8 V64:$Vn))),
5715 (REV64v8i8 (COPY_TO_REGCLASS V64:$Vn, GPR64))>;
5716 def : Pat<(i64 (bitconvert (v4i16 V64:$Vn))),
5717 (REV64v4i16 (COPY_TO_REGCLASS V64:$Vn, GPR64))>;
5718 def : Pat<(i64 (bitconvert (v2i32 V64:$Vn))),
5719 (REV64v2i32 (COPY_TO_REGCLASS V64:$Vn, GPR64))>;
5720 def : Pat<(i64 (bitconvert (v4f16 V64:$Vn))),
5721 (REV64v4i16 (COPY_TO_REGCLASS V64:$Vn, GPR64))>;
5722 def : Pat<(i64 (bitconvert (v2f32 V64:$Vn))),
5723 (REV64v2i32 (COPY_TO_REGCLASS V64:$Vn, GPR64))>;
5725 def : Pat<(v1i64 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5726 def : Pat<(v1f64 (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5727 def : Pat<(i64 (bitconvert (v1i64 V64:$Vn))),
5728 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5729 def : Pat<(v1i64 (scalar_to_vector GPR64:$Xn)),
5730 (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5731 def : Pat<(v1f64 (scalar_to_vector GPR64:$Xn)),
5732 (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5733 def : Pat<(v1f64 (scalar_to_vector (f64 FPR64:$Xn))), (v1f64 FPR64:$Xn)>;
5735 def : Pat<(f32 (bitconvert (i32 GPR32:$Xn))),
5736 (COPY_TO_REGCLASS GPR32:$Xn, FPR32)>;
5737 def : Pat<(i32 (bitconvert (f32 FPR32:$Xn))),
5738 (COPY_TO_REGCLASS FPR32:$Xn, GPR32)>;
5739 def : Pat<(f64 (bitconvert (i64 GPR64:$Xn))),
5740 (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
5741 def : Pat<(i64 (bitconvert (f64 FPR64:$Xn))),
5742 (COPY_TO_REGCLASS FPR64:$Xn, GPR64)>;
5743 def : Pat<(i64 (bitconvert (v1f64 V64:$Vn))),
5744 (COPY_TO_REGCLASS V64:$Vn, GPR64)>;
5746 let Predicates = [IsLE] in {
5747 def : Pat<(v1i64 (bitconvert (v2i32 FPR64:$src))), (v1i64 FPR64:$src)>;
5748 def : Pat<(v1i64 (bitconvert (v4i16 FPR64:$src))), (v1i64 FPR64:$src)>;
5749 def : Pat<(v1i64 (bitconvert (v8i8 FPR64:$src))), (v1i64 FPR64:$src)>;
5750 def : Pat<(v1i64 (bitconvert (v4f16 FPR64:$src))), (v1i64 FPR64:$src)>;
5751 def : Pat<(v1i64 (bitconvert (v2f32 FPR64:$src))), (v1i64 FPR64:$src)>;
5753 let Predicates = [IsBE] in {
5754 def : Pat<(v1i64 (bitconvert (v2i32 FPR64:$src))),
5755 (v1i64 (REV64v2i32 FPR64:$src))>;
5756 def : Pat<(v1i64 (bitconvert (v4i16 FPR64:$src))),
5757 (v1i64 (REV64v4i16 FPR64:$src))>;
5758 def : Pat<(v1i64 (bitconvert (v8i8 FPR64:$src))),
5759 (v1i64 (REV64v8i8 FPR64:$src))>;
5760 def : Pat<(v1i64 (bitconvert (v4f16 FPR64:$src))),
5761 (v1i64 (REV64v4i16 FPR64:$src))>;
5762 def : Pat<(v1i64 (bitconvert (v2f32 FPR64:$src))),
5763 (v1i64 (REV64v2i32 FPR64:$src))>;
5765 def : Pat<(v1i64 (bitconvert (v1f64 FPR64:$src))), (v1i64 FPR64:$src)>;
5766 def : Pat<(v1i64 (bitconvert (f64 FPR64:$src))), (v1i64 FPR64:$src)>;
5768 let Predicates = [IsLE] in {
5769 def : Pat<(v2i32 (bitconvert (v1i64 FPR64:$src))), (v2i32 FPR64:$src)>;
5770 def : Pat<(v2i32 (bitconvert (v4i16 FPR64:$src))), (v2i32 FPR64:$src)>;
5771 def : Pat<(v2i32 (bitconvert (v8i8 FPR64:$src))), (v2i32 FPR64:$src)>;
5772 def : Pat<(v2i32 (bitconvert (f64 FPR64:$src))), (v2i32 FPR64:$src)>;
5773 def : Pat<(v2i32 (bitconvert (v1f64 FPR64:$src))), (v2i32 FPR64:$src)>;
5774 def : Pat<(v2i32 (bitconvert (v4f16 FPR64:$src))), (v2i32 FPR64:$src)>;
5776 let Predicates = [IsBE] in {
5777 def : Pat<(v2i32 (bitconvert (v1i64 FPR64:$src))),
5778 (v2i32 (REV64v2i32 FPR64:$src))>;
5779 def : Pat<(v2i32 (bitconvert (v4i16 FPR64:$src))),
5780 (v2i32 (REV32v4i16 FPR64:$src))>;
5781 def : Pat<(v2i32 (bitconvert (v8i8 FPR64:$src))),
5782 (v2i32 (REV32v8i8 FPR64:$src))>;
5783 def : Pat<(v2i32 (bitconvert (f64 FPR64:$src))),
5784 (v2i32 (REV64v2i32 FPR64:$src))>;
5785 def : Pat<(v2i32 (bitconvert (v1f64 FPR64:$src))),
5786 (v2i32 (REV64v2i32 FPR64:$src))>;
5787 def : Pat<(v2i32 (bitconvert (v4f16 FPR64:$src))),
5788 (v2i32 (REV64v4i16 FPR64:$src))>;
5790 def : Pat<(v2i32 (bitconvert (v2f32 FPR64:$src))), (v2i32 FPR64:$src)>;
5792 let Predicates = [IsLE] in {
5793 def : Pat<(v4i16 (bitconvert (v1i64 FPR64:$src))), (v4i16 FPR64:$src)>;
5794 def : Pat<(v4i16 (bitconvert (v2i32 FPR64:$src))), (v4i16 FPR64:$src)>;
5795 def : Pat<(v4i16 (bitconvert (v8i8 FPR64:$src))), (v4i16 FPR64:$src)>;
5796 def : Pat<(v4i16 (bitconvert (f64 FPR64:$src))), (v4i16 FPR64:$src)>;
5797 def : Pat<(v4i16 (bitconvert (v4f16 FPR64:$src))), (v4i16 FPR64:$src)>;
5798 def : Pat<(v4i16 (bitconvert (v2f32 FPR64:$src))), (v4i16 FPR64:$src)>;
5799 def : Pat<(v4i16 (bitconvert (v1f64 FPR64:$src))), (v4i16 FPR64:$src)>;
5801 let Predicates = [IsBE] in {
5802 def : Pat<(v4i16 (bitconvert (v1i64 FPR64:$src))),
5803 (v4i16 (REV64v4i16 FPR64:$src))>;
5804 def : Pat<(v4i16 (bitconvert (v2i32 FPR64:$src))),
5805 (v4i16 (REV32v4i16 FPR64:$src))>;
5806 def : Pat<(v4i16 (bitconvert (v8i8 FPR64:$src))),
5807 (v4i16 (REV16v8i8 FPR64:$src))>;
5808 def : Pat<(v4i16 (bitconvert (f64 FPR64:$src))),
5809 (v4i16 (REV64v4i16 FPR64:$src))>;
5810 def : Pat<(v4i16 (bitconvert (v4f16 FPR64:$src))),
5811 (v4i16 (REV32v4i16 FPR64:$src))>;
5812 def : Pat<(v4i16 (bitconvert (v2f32 FPR64:$src))),
5813 (v4i16 (REV32v4i16 FPR64:$src))>;
5814 def : Pat<(v4i16 (bitconvert (v1f64 FPR64:$src))),
5815 (v4i16 (REV64v4i16 FPR64:$src))>;
5818 let Predicates = [IsLE] in {
5819 def : Pat<(v4f16 (bitconvert (v1i64 FPR64:$src))), (v4f16 FPR64:$src)>;
5820 def : Pat<(v4f16 (bitconvert (v2i32 FPR64:$src))), (v4f16 FPR64:$src)>;
5821 def : Pat<(v4f16 (bitconvert (v4i16 FPR64:$src))), (v4f16 FPR64:$src)>;
5822 def : Pat<(v4f16 (bitconvert (v8i8 FPR64:$src))), (v4f16 FPR64:$src)>;
5823 def : Pat<(v4f16 (bitconvert (f64 FPR64:$src))), (v4f16 FPR64:$src)>;
5824 def : Pat<(v4f16 (bitconvert (v2f32 FPR64:$src))), (v4f16 FPR64:$src)>;
5825 def : Pat<(v4f16 (bitconvert (v1f64 FPR64:$src))), (v4f16 FPR64:$src)>;
5827 let Predicates = [IsBE] in {
5828 def : Pat<(v4f16 (bitconvert (v1i64 FPR64:$src))),
5829 (v4f16 (REV64v4i16 FPR64:$src))>;
5830 def : Pat<(v4f16 (bitconvert (v2i32 FPR64:$src))),
5831 (v4f16 (REV64v4i16 FPR64:$src))>;
5832 def : Pat<(v4f16 (bitconvert (v4i16 FPR64:$src))),
5833 (v4f16 (REV64v4i16 FPR64:$src))>;
5834 def : Pat<(v4f16 (bitconvert (v8i8 FPR64:$src))),
5835 (v4f16 (REV16v8i8 FPR64:$src))>;
5836 def : Pat<(v4f16 (bitconvert (f64 FPR64:$src))),
5837 (v4f16 (REV64v4i16 FPR64:$src))>;
5838 def : Pat<(v4f16 (bitconvert (v2f32 FPR64:$src))),
5839 (v4f16 (REV64v4i16 FPR64:$src))>;
5840 def : Pat<(v4f16 (bitconvert (v1f64 FPR64:$src))),
5841 (v4f16 (REV64v4i16 FPR64:$src))>;
5846 let Predicates = [IsLE] in {
5847 def : Pat<(v8i8 (bitconvert (v1i64 FPR64:$src))), (v8i8 FPR64:$src)>;
5848 def : Pat<(v8i8 (bitconvert (v2i32 FPR64:$src))), (v8i8 FPR64:$src)>;
5849 def : Pat<(v8i8 (bitconvert (v4i16 FPR64:$src))), (v8i8 FPR64:$src)>;
5850 def : Pat<(v8i8 (bitconvert (f64 FPR64:$src))), (v8i8 FPR64:$src)>;
5851 def : Pat<(v8i8 (bitconvert (v2f32 FPR64:$src))), (v8i8 FPR64:$src)>;
5852 def : Pat<(v8i8 (bitconvert (v1f64 FPR64:$src))), (v8i8 FPR64:$src)>;
5853 def : Pat<(v8i8 (bitconvert (v4f16 FPR64:$src))), (v8i8 FPR64:$src)>;
5855 let Predicates = [IsBE] in {
5856 def : Pat<(v8i8 (bitconvert (v1i64 FPR64:$src))),
5857 (v8i8 (REV64v8i8 FPR64:$src))>;
5858 def : Pat<(v8i8 (bitconvert (v2i32 FPR64:$src))),
5859 (v8i8 (REV32v8i8 FPR64:$src))>;
5860 def : Pat<(v8i8 (bitconvert (v4i16 FPR64:$src))),
5861 (v8i8 (REV16v8i8 FPR64:$src))>;
5862 def : Pat<(v8i8 (bitconvert (f64 FPR64:$src))),
5863 (v8i8 (REV64v8i8 FPR64:$src))>;
5864 def : Pat<(v8i8 (bitconvert (v2f32 FPR64:$src))),
5865 (v8i8 (REV32v8i8 FPR64:$src))>;
5866 def : Pat<(v8i8 (bitconvert (v1f64 FPR64:$src))),
5867 (v8i8 (REV64v8i8 FPR64:$src))>;
5868 def : Pat<(v8i8 (bitconvert (v4f16 FPR64:$src))),
5869 (v8i8 (REV16v8i8 FPR64:$src))>;
5872 let Predicates = [IsLE] in {
5873 def : Pat<(f64 (bitconvert (v2i32 FPR64:$src))), (f64 FPR64:$src)>;
5874 def : Pat<(f64 (bitconvert (v4i16 FPR64:$src))), (f64 FPR64:$src)>;
5875 def : Pat<(f64 (bitconvert (v2f32 FPR64:$src))), (f64 FPR64:$src)>;
5876 def : Pat<(f64 (bitconvert (v8i8 FPR64:$src))), (f64 FPR64:$src)>;
5877 def : Pat<(f64 (bitconvert (v4f16 FPR64:$src))), (f64 FPR64:$src)>;
5879 let Predicates = [IsBE] in {
5880 def : Pat<(f64 (bitconvert (v2i32 FPR64:$src))),
5881 (f64 (REV64v2i32 FPR64:$src))>;
5882 def : Pat<(f64 (bitconvert (v4i16 FPR64:$src))),
5883 (f64 (REV64v4i16 FPR64:$src))>;
5884 def : Pat<(f64 (bitconvert (v2f32 FPR64:$src))),
5885 (f64 (REV64v2i32 FPR64:$src))>;
5886 def : Pat<(f64 (bitconvert (v8i8 FPR64:$src))),
5887 (f64 (REV64v8i8 FPR64:$src))>;
5888 def : Pat<(f64 (bitconvert (v4f16 FPR64:$src))),
5889 (f64 (REV64v4i16 FPR64:$src))>;
5891 def : Pat<(f64 (bitconvert (v1i64 FPR64:$src))), (f64 FPR64:$src)>;
5892 def : Pat<(f64 (bitconvert (v1f64 FPR64:$src))), (f64 FPR64:$src)>;
5894 let Predicates = [IsLE] in {
5895 def : Pat<(v1f64 (bitconvert (v2i32 FPR64:$src))), (v1f64 FPR64:$src)>;
5896 def : Pat<(v1f64 (bitconvert (v4i16 FPR64:$src))), (v1f64 FPR64:$src)>;
5897 def : Pat<(v1f64 (bitconvert (v8i8 FPR64:$src))), (v1f64 FPR64:$src)>;
5898 def : Pat<(v1f64 (bitconvert (v2f32 FPR64:$src))), (v1f64 FPR64:$src)>;
5899 def : Pat<(v1f64 (bitconvert (v4f16 FPR64:$src))), (v1f64 FPR64:$src)>;
5901 let Predicates = [IsBE] in {
5902 def : Pat<(v1f64 (bitconvert (v2i32 FPR64:$src))),
5903 (v1f64 (REV64v2i32 FPR64:$src))>;
5904 def : Pat<(v1f64 (bitconvert (v4i16 FPR64:$src))),
5905 (v1f64 (REV64v4i16 FPR64:$src))>;
5906 def : Pat<(v1f64 (bitconvert (v8i8 FPR64:$src))),
5907 (v1f64 (REV64v8i8 FPR64:$src))>;
5908 def : Pat<(v1f64 (bitconvert (v2f32 FPR64:$src))),
5909 (v1f64 (REV64v2i32 FPR64:$src))>;
5910 def : Pat<(v1f64 (bitconvert (v4f16 FPR64:$src))),
5911 (v1f64 (REV64v4i16 FPR64:$src))>;
5913 def : Pat<(v1f64 (bitconvert (v1i64 FPR64:$src))), (v1f64 FPR64:$src)>;
5914 def : Pat<(v1f64 (bitconvert (f64 FPR64:$src))), (v1f64 FPR64:$src)>;
5916 let Predicates = [IsLE] in {
5917 def : Pat<(v2f32 (bitconvert (v1i64 FPR64:$src))), (v2f32 FPR64:$src)>;
5918 def : Pat<(v2f32 (bitconvert (v4i16 FPR64:$src))), (v2f32 FPR64:$src)>;
5919 def : Pat<(v2f32 (bitconvert (v8i8 FPR64:$src))), (v2f32 FPR64:$src)>;
5920 def : Pat<(v2f32 (bitconvert (v1f64 FPR64:$src))), (v2f32 FPR64:$src)>;
5921 def : Pat<(v2f32 (bitconvert (f64 FPR64:$src))), (v2f32 FPR64:$src)>;
5922 def : Pat<(v2f32 (bitconvert (v4f16 FPR64:$src))), (v2f32 FPR64:$src)>;
5924 let Predicates = [IsBE] in {
5925 def : Pat<(v2f32 (bitconvert (v1i64 FPR64:$src))),
5926 (v2f32 (REV64v2i32 FPR64:$src))>;
5927 def : Pat<(v2f32 (bitconvert (v4i16 FPR64:$src))),
5928 (v2f32 (REV32v4i16 FPR64:$src))>;
5929 def : Pat<(v2f32 (bitconvert (v8i8 FPR64:$src))),
5930 (v2f32 (REV32v8i8 FPR64:$src))>;
5931 def : Pat<(v2f32 (bitconvert (v1f64 FPR64:$src))),
5932 (v2f32 (REV64v2i32 FPR64:$src))>;
5933 def : Pat<(v2f32 (bitconvert (f64 FPR64:$src))),
5934 (v2f32 (REV64v2i32 FPR64:$src))>;
5935 def : Pat<(v2f32 (bitconvert (v4f16 FPR64:$src))),
5936 (v2f32 (REV64v4i16 FPR64:$src))>;
5938 def : Pat<(v2f32 (bitconvert (v2i32 FPR64:$src))), (v2f32 FPR64:$src)>;
5940 let Predicates = [IsLE] in {
5941 def : Pat<(f128 (bitconvert (v2i64 FPR128:$src))), (f128 FPR128:$src)>;
5942 def : Pat<(f128 (bitconvert (v4i32 FPR128:$src))), (f128 FPR128:$src)>;
5943 def : Pat<(f128 (bitconvert (v8i16 FPR128:$src))), (f128 FPR128:$src)>;
5944 def : Pat<(f128 (bitconvert (v2f64 FPR128:$src))), (f128 FPR128:$src)>;
5945 def : Pat<(f128 (bitconvert (v4f32 FPR128:$src))), (f128 FPR128:$src)>;
5946 def : Pat<(f128 (bitconvert (v8f16 FPR128:$src))), (f128 FPR128:$src)>;
5947 def : Pat<(f128 (bitconvert (v16i8 FPR128:$src))), (f128 FPR128:$src)>;
5949 let Predicates = [IsBE] in {
5950 def : Pat<(f128 (bitconvert (v2i64 FPR128:$src))),
5951 (f128 (EXTv16i8 FPR128:$src, FPR128:$src, (i32 8)))>;
5952 def : Pat<(f128 (bitconvert (v4i32 FPR128:$src))),
5953 (f128 (EXTv16i8 (REV64v4i32 FPR128:$src),
5954 (REV64v4i32 FPR128:$src), (i32 8)))>;
5955 def : Pat<(f128 (bitconvert (v8i16 FPR128:$src))),
5956 (f128 (EXTv16i8 (REV64v8i16 FPR128:$src),
5957 (REV64v8i16 FPR128:$src), (i32 8)))>;
5958 def : Pat<(f128 (bitconvert (v8f16 FPR128:$src))),
5959 (f128 (EXTv16i8 (REV64v8i16 FPR128:$src),
5960 (REV64v8i16 FPR128:$src), (i32 8)))>;
5961 def : Pat<(f128 (bitconvert (v2f64 FPR128:$src))),
5962 (f128 (EXTv16i8 FPR128:$src, FPR128:$src, (i32 8)))>;
5963 def : Pat<(f128 (bitconvert (v4f32 FPR128:$src))),
5964 (f128 (EXTv16i8 (REV64v4i32 FPR128:$src),
5965 (REV64v4i32 FPR128:$src), (i32 8)))>;
5966 def : Pat<(f128 (bitconvert (v16i8 FPR128:$src))),
5967 (f128 (EXTv16i8 (REV64v16i8 FPR128:$src),
5968 (REV64v16i8 FPR128:$src), (i32 8)))>;
5971 let Predicates = [IsLE] in {
5972 def : Pat<(v2f64 (bitconvert (f128 FPR128:$src))), (v2f64 FPR128:$src)>;
5973 def : Pat<(v2f64 (bitconvert (v4i32 FPR128:$src))), (v2f64 FPR128:$src)>;
5974 def : Pat<(v2f64 (bitconvert (v8i16 FPR128:$src))), (v2f64 FPR128:$src)>;
5975 def : Pat<(v2f64 (bitconvert (v8f16 FPR128:$src))), (v2f64 FPR128:$src)>;
5976 def : Pat<(v2f64 (bitconvert (v16i8 FPR128:$src))), (v2f64 FPR128:$src)>;
5977 def : Pat<(v2f64 (bitconvert (v4f32 FPR128:$src))), (v2f64 FPR128:$src)>;
5979 let Predicates = [IsBE] in {
5980 def : Pat<(v2f64 (bitconvert (f128 FPR128:$src))),
5981 (v2f64 (EXTv16i8 FPR128:$src,
5982 FPR128:$src, (i32 8)))>;
5983 def : Pat<(v2f64 (bitconvert (v4i32 FPR128:$src))),
5984 (v2f64 (REV64v4i32 FPR128:$src))>;
5985 def : Pat<(v2f64 (bitconvert (v8i16 FPR128:$src))),
5986 (v2f64 (REV64v8i16 FPR128:$src))>;
5987 def : Pat<(v2f64 (bitconvert (v8f16 FPR128:$src))),
5988 (v2f64 (REV64v8i16 FPR128:$src))>;
5989 def : Pat<(v2f64 (bitconvert (v16i8 FPR128:$src))),
5990 (v2f64 (REV64v16i8 FPR128:$src))>;
5991 def : Pat<(v2f64 (bitconvert (v4f32 FPR128:$src))),
5992 (v2f64 (REV64v4i32 FPR128:$src))>;
5994 def : Pat<(v2f64 (bitconvert (v2i64 FPR128:$src))), (v2f64 FPR128:$src)>;
5996 let Predicates = [IsLE] in {
5997 def : Pat<(v4f32 (bitconvert (f128 FPR128:$src))), (v4f32 FPR128:$src)>;
5998 def : Pat<(v4f32 (bitconvert (v8i16 FPR128:$src))), (v4f32 FPR128:$src)>;
5999 def : Pat<(v4f32 (bitconvert (v8f16 FPR128:$src))), (v4f32 FPR128:$src)>;
6000 def : Pat<(v4f32 (bitconvert (v16i8 FPR128:$src))), (v4f32 FPR128:$src)>;
6001 def : Pat<(v4f32 (bitconvert (v2i64 FPR128:$src))), (v4f32 FPR128:$src)>;
6002 def : Pat<(v4f32 (bitconvert (v2f64 FPR128:$src))), (v4f32 FPR128:$src)>;
6004 let Predicates = [IsBE] in {
6005 def : Pat<(v4f32 (bitconvert (f128 FPR128:$src))),
6006 (v4f32 (EXTv16i8 (REV64v4i32 FPR128:$src),
6007 (REV64v4i32 FPR128:$src), (i32 8)))>;
6008 def : Pat<(v4f32 (bitconvert (v8i16 FPR128:$src))),
6009 (v4f32 (REV32v8i16 FPR128:$src))>;
6010 def : Pat<(v4f32 (bitconvert (v8f16 FPR128:$src))),
6011 (v4f32 (REV32v8i16 FPR128:$src))>;
6012 def : Pat<(v4f32 (bitconvert (v16i8 FPR128:$src))),
6013 (v4f32 (REV32v16i8 FPR128:$src))>;
6014 def : Pat<(v4f32 (bitconvert (v2i64 FPR128:$src))),
6015 (v4f32 (REV64v4i32 FPR128:$src))>;
6016 def : Pat<(v4f32 (bitconvert (v2f64 FPR128:$src))),
6017 (v4f32 (REV64v4i32 FPR128:$src))>;
6019 def : Pat<(v4f32 (bitconvert (v4i32 FPR128:$src))), (v4f32 FPR128:$src)>;
6021 let Predicates = [IsLE] in {
6022 def : Pat<(v2i64 (bitconvert (f128 FPR128:$src))), (v2i64 FPR128:$src)>;
6023 def : Pat<(v2i64 (bitconvert (v4i32 FPR128:$src))), (v2i64 FPR128:$src)>;
6024 def : Pat<(v2i64 (bitconvert (v8i16 FPR128:$src))), (v2i64 FPR128:$src)>;
6025 def : Pat<(v2i64 (bitconvert (v16i8 FPR128:$src))), (v2i64 FPR128:$src)>;
6026 def : Pat<(v2i64 (bitconvert (v4f32 FPR128:$src))), (v2i64 FPR128:$src)>;
6027 def : Pat<(v2i64 (bitconvert (v8f16 FPR128:$src))), (v2i64 FPR128:$src)>;
6029 let Predicates = [IsBE] in {
6030 def : Pat<(v2i64 (bitconvert (f128 FPR128:$src))),
6031 (v2i64 (EXTv16i8 FPR128:$src,
6032 FPR128:$src, (i32 8)))>;
6033 def : Pat<(v2i64 (bitconvert (v4i32 FPR128:$src))),
6034 (v2i64 (REV64v4i32 FPR128:$src))>;
6035 def : Pat<(v2i64 (bitconvert (v8i16 FPR128:$src))),
6036 (v2i64 (REV64v8i16 FPR128:$src))>;
6037 def : Pat<(v2i64 (bitconvert (v16i8 FPR128:$src))),
6038 (v2i64 (REV64v16i8 FPR128:$src))>;
6039 def : Pat<(v2i64 (bitconvert (v4f32 FPR128:$src))),
6040 (v2i64 (REV64v4i32 FPR128:$src))>;
6041 def : Pat<(v2i64 (bitconvert (v8f16 FPR128:$src))),
6042 (v2i64 (REV64v8i16 FPR128:$src))>;
6044 def : Pat<(v2i64 (bitconvert (v2f64 FPR128:$src))), (v2i64 FPR128:$src)>;
6046 let Predicates = [IsLE] in {
6047 def : Pat<(v4i32 (bitconvert (f128 FPR128:$src))), (v4i32 FPR128:$src)>;
6048 def : Pat<(v4i32 (bitconvert (v2i64 FPR128:$src))), (v4i32 FPR128:$src)>;
6049 def : Pat<(v4i32 (bitconvert (v8i16 FPR128:$src))), (v4i32 FPR128:$src)>;
6050 def : Pat<(v4i32 (bitconvert (v16i8 FPR128:$src))), (v4i32 FPR128:$src)>;
6051 def : Pat<(v4i32 (bitconvert (v2f64 FPR128:$src))), (v4i32 FPR128:$src)>;
6052 def : Pat<(v4i32 (bitconvert (v8f16 FPR128:$src))), (v4i32 FPR128:$src)>;
6054 let Predicates = [IsBE] in {
6055 def : Pat<(v4i32 (bitconvert (f128 FPR128:$src))),
6056 (v4i32 (EXTv16i8 (REV64v4i32 FPR128:$src),
6057 (REV64v4i32 FPR128:$src),
6059 def : Pat<(v4i32 (bitconvert (v2i64 FPR128:$src))),
6060 (v4i32 (REV64v4i32 FPR128:$src))>;
6061 def : Pat<(v4i32 (bitconvert (v8i16 FPR128:$src))),
6062 (v4i32 (REV32v8i16 FPR128:$src))>;
6063 def : Pat<(v4i32 (bitconvert (v16i8 FPR128:$src))),
6064 (v4i32 (REV32v16i8 FPR128:$src))>;
6065 def : Pat<(v4i32 (bitconvert (v2f64 FPR128:$src))),
6066 (v4i32 (REV64v4i32 FPR128:$src))>;
6067 def : Pat<(v4i32 (bitconvert (v8f16 FPR128:$src))),
6068 (v4i32 (REV32v8i16 FPR128:$src))>;
6070 def : Pat<(v4i32 (bitconvert (v4f32 FPR128:$src))), (v4i32 FPR128:$src)>;
6072 let Predicates = [IsLE] in {
6073 def : Pat<(v8i16 (bitconvert (f128 FPR128:$src))), (v8i16 FPR128:$src)>;
6074 def : Pat<(v8i16 (bitconvert (v2i64 FPR128:$src))), (v8i16 FPR128:$src)>;
6075 def : Pat<(v8i16 (bitconvert (v4i32 FPR128:$src))), (v8i16 FPR128:$src)>;
6076 def : Pat<(v8i16 (bitconvert (v16i8 FPR128:$src))), (v8i16 FPR128:$src)>;
6077 def : Pat<(v8i16 (bitconvert (v2f64 FPR128:$src))), (v8i16 FPR128:$src)>;
6078 def : Pat<(v8i16 (bitconvert (v4f32 FPR128:$src))), (v8i16 FPR128:$src)>;
6079 def : Pat<(v8i16 (bitconvert (v8f16 FPR128:$src))), (v8i16 FPR128:$src)>;
6081 let Predicates = [IsBE] in {
6082 def : Pat<(v8i16 (bitconvert (f128 FPR128:$src))),
6083 (v8i16 (EXTv16i8 (REV64v8i16 FPR128:$src),
6084 (REV64v8i16 FPR128:$src),
6086 def : Pat<(v8i16 (bitconvert (v2i64 FPR128:$src))),
6087 (v8i16 (REV64v8i16 FPR128:$src))>;
6088 def : Pat<(v8i16 (bitconvert (v4i32 FPR128:$src))),
6089 (v8i16 (REV32v8i16 FPR128:$src))>;
6090 def : Pat<(v8i16 (bitconvert (v16i8 FPR128:$src))),
6091 (v8i16 (REV16v16i8 FPR128:$src))>;
6092 def : Pat<(v8i16 (bitconvert (v2f64 FPR128:$src))),
6093 (v8i16 (REV64v8i16 FPR128:$src))>;
6094 def : Pat<(v8i16 (bitconvert (v4f32 FPR128:$src))),
6095 (v8i16 (REV32v8i16 FPR128:$src))>;
6096 def : Pat<(v8i16 (bitconvert (v8f16 FPR128:$src))),
6097 (v8i16 (REV32v8i16 FPR128:$src))>;
6100 let Predicates = [IsLE] in {
6101 def : Pat<(v8f16 (bitconvert (f128 FPR128:$src))), (v8f16 FPR128:$src)>;
6102 def : Pat<(v8f16 (bitconvert (v2i64 FPR128:$src))), (v8f16 FPR128:$src)>;
6103 def : Pat<(v8f16 (bitconvert (v4i32 FPR128:$src))), (v8f16 FPR128:$src)>;
6104 def : Pat<(v8f16 (bitconvert (v8i16 FPR128:$src))), (v8f16 FPR128:$src)>;
6105 def : Pat<(v8f16 (bitconvert (v16i8 FPR128:$src))), (v8f16 FPR128:$src)>;
6106 def : Pat<(v8f16 (bitconvert (v2f64 FPR128:$src))), (v8f16 FPR128:$src)>;
6107 def : Pat<(v8f16 (bitconvert (v4f32 FPR128:$src))), (v8f16 FPR128:$src)>;
6109 let Predicates = [IsBE] in {
6110 def : Pat<(v8f16 (bitconvert (f128 FPR128:$src))),
6111 (v8f16 (EXTv16i8 (REV64v8i16 FPR128:$src),
6112 (REV64v8i16 FPR128:$src),
6114 def : Pat<(v8f16 (bitconvert (v2i64 FPR128:$src))),
6115 (v8f16 (REV64v8i16 FPR128:$src))>;
6116 def : Pat<(v8f16 (bitconvert (v4i32 FPR128:$src))),
6117 (v8f16 (REV32v8i16 FPR128:$src))>;
6118 def : Pat<(v8f16 (bitconvert (v8i16 FPR128:$src))),
6119 (v8f16 (REV64v8i16 FPR128:$src))>;
6120 def : Pat<(v8f16 (bitconvert (v16i8 FPR128:$src))),
6121 (v8f16 (REV16v16i8 FPR128:$src))>;
6122 def : Pat<(v8f16 (bitconvert (v2f64 FPR128:$src))),
6123 (v8f16 (REV64v8i16 FPR128:$src))>;
6124 def : Pat<(v8f16 (bitconvert (v4f32 FPR128:$src))),
6125 (v8f16 (REV32v8i16 FPR128:$src))>;
6128 let Predicates = [IsLE] in {
6129 def : Pat<(v16i8 (bitconvert (f128 FPR128:$src))), (v16i8 FPR128:$src)>;
6130 def : Pat<(v16i8 (bitconvert (v2i64 FPR128:$src))), (v16i8 FPR128:$src)>;
6131 def : Pat<(v16i8 (bitconvert (v4i32 FPR128:$src))), (v16i8 FPR128:$src)>;
6132 def : Pat<(v16i8 (bitconvert (v8i16 FPR128:$src))), (v16i8 FPR128:$src)>;
6133 def : Pat<(v16i8 (bitconvert (v2f64 FPR128:$src))), (v16i8 FPR128:$src)>;
6134 def : Pat<(v16i8 (bitconvert (v4f32 FPR128:$src))), (v16i8 FPR128:$src)>;
6135 def : Pat<(v16i8 (bitconvert (v8f16 FPR128:$src))), (v16i8 FPR128:$src)>;
6137 let Predicates = [IsBE] in {
6138 def : Pat<(v16i8 (bitconvert (f128 FPR128:$src))),
6139 (v16i8 (EXTv16i8 (REV64v16i8 FPR128:$src),
6140 (REV64v16i8 FPR128:$src),
6142 def : Pat<(v16i8 (bitconvert (v2i64 FPR128:$src))),
6143 (v16i8 (REV64v16i8 FPR128:$src))>;
6144 def : Pat<(v16i8 (bitconvert (v4i32 FPR128:$src))),
6145 (v16i8 (REV32v16i8 FPR128:$src))>;
6146 def : Pat<(v16i8 (bitconvert (v8i16 FPR128:$src))),
6147 (v16i8 (REV16v16i8 FPR128:$src))>;
6148 def : Pat<(v16i8 (bitconvert (v2f64 FPR128:$src))),
6149 (v16i8 (REV64v16i8 FPR128:$src))>;
6150 def : Pat<(v16i8 (bitconvert (v4f32 FPR128:$src))),
6151 (v16i8 (REV32v16i8 FPR128:$src))>;
6152 def : Pat<(v16i8 (bitconvert (v8f16 FPR128:$src))),
6153 (v16i8 (REV16v16i8 FPR128:$src))>;
6156 def : Pat<(v4i16 (extract_subvector V128:$Rn, (i64 0))),
6157 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6158 def : Pat<(v8i8 (extract_subvector V128:$Rn, (i64 0))),
6159 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6160 def : Pat<(v2f32 (extract_subvector V128:$Rn, (i64 0))),
6161 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6162 def : Pat<(v4f16 (extract_subvector V128:$Rn, (i64 0))),
6163 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6164 def : Pat<(v2i32 (extract_subvector V128:$Rn, (i64 0))),
6165 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6166 def : Pat<(v1i64 (extract_subvector V128:$Rn, (i64 0))),
6167 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6168 def : Pat<(v1f64 (extract_subvector V128:$Rn, (i64 0))),
6169 (EXTRACT_SUBREG V128:$Rn, dsub)>;
6171 def : Pat<(v8i8 (extract_subvector (v16i8 FPR128:$Rn), (i64 1))),
6172 (EXTRACT_SUBREG (DUPv2i64lane FPR128:$Rn, 1), dsub)>;
6173 def : Pat<(v4i16 (extract_subvector (v8i16 FPR128:$Rn), (i64 1))),
6174 (EXTRACT_SUBREG (DUPv2i64lane FPR128:$Rn, 1), dsub)>;
6175 def : Pat<(v2i32 (extract_subvector (v4i32 FPR128:$Rn), (i64 1))),
6176 (EXTRACT_SUBREG (DUPv2i64lane FPR128:$Rn, 1), dsub)>;
6177 def : Pat<(v1i64 (extract_subvector (v2i64 FPR128:$Rn), (i64 1))),
6178 (EXTRACT_SUBREG (DUPv2i64lane FPR128:$Rn, 1), dsub)>;
6180 // A 64-bit subvector insert to the first 128-bit vector position
6181 // is a subregister copy that needs no instruction.
6182 def : Pat<(insert_subvector undef, (v1i64 FPR64:$src), (i32 0)),
6183 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6184 def : Pat<(insert_subvector undef, (v1f64 FPR64:$src), (i32 0)),
6185 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6186 def : Pat<(insert_subvector undef, (v2i32 FPR64:$src), (i32 0)),
6187 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6188 def : Pat<(insert_subvector undef, (v2f32 FPR64:$src), (i32 0)),
6189 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6190 def : Pat<(insert_subvector undef, (v4i16 FPR64:$src), (i32 0)),
6191 (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6192 def : Pat<(insert_subvector undef, (v4f16 FPR64:$src), (i32 0)),
6193 (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6194 def : Pat<(insert_subvector undef, (v8i8 FPR64:$src), (i32 0)),
6195 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
6197 // Use pair-wise add instructions when summing up the lanes for v2f64, v2i64
6199 def : Pat<(i64 (add (vector_extract (v2i64 FPR128:$Rn), (i64 0)),
6200 (vector_extract (v2i64 FPR128:$Rn), (i64 1)))),
6201 (i64 (ADDPv2i64p (v2i64 FPR128:$Rn)))>;
6202 def : Pat<(f64 (fadd (vector_extract (v2f64 FPR128:$Rn), (i64 0)),
6203 (vector_extract (v2f64 FPR128:$Rn), (i64 1)))),
6204 (f64 (FADDPv2i64p (v2f64 FPR128:$Rn)))>;
6205 // vector_extract on 64-bit vectors gets promoted to a 128 bit vector,
6206 // so we match on v4f32 here, not v2f32. This will also catch adding
6207 // the low two lanes of a true v4f32 vector.
6208 def : Pat<(fadd (vector_extract (v4f32 FPR128:$Rn), (i64 0)),
6209 (vector_extract (v4f32 FPR128:$Rn), (i64 1))),
6210 (f32 (FADDPv2i32p (EXTRACT_SUBREG FPR128:$Rn, dsub)))>;
6212 // Scalar 64-bit shifts in FPR64 registers.
6213 def : Pat<(i64 (int_aarch64_neon_sshl (i64 FPR64:$Rn), (i64 FPR64:$Rm))),
6214 (SSHLv1i64 FPR64:$Rn, FPR64:$Rm)>;
6215 def : Pat<(i64 (int_aarch64_neon_ushl (i64 FPR64:$Rn), (i64 FPR64:$Rm))),
6216 (USHLv1i64 FPR64:$Rn, FPR64:$Rm)>;
6217 def : Pat<(i64 (int_aarch64_neon_srshl (i64 FPR64:$Rn), (i64 FPR64:$Rm))),
6218 (SRSHLv1i64 FPR64:$Rn, FPR64:$Rm)>;
6219 def : Pat<(i64 (int_aarch64_neon_urshl (i64 FPR64:$Rn), (i64 FPR64:$Rm))),
6220 (URSHLv1i64 FPR64:$Rn, FPR64:$Rm)>;
6222 // Patterns for nontemporal/no-allocate stores.
6223 // We have to resort to tricks to turn a single-input store into a store pair,
6224 // because there is no single-input nontemporal store, only STNP.
6225 let Predicates = [IsLE] in {
6226 let AddedComplexity = 15 in {
6227 class NTStore128Pat<ValueType VT> :
6228 Pat<(nontemporalstore (VT FPR128:$Rt),
6229 (am_indexed7s64 GPR64sp:$Rn, simm7s8:$offset)),
6230 (STNPDi (EXTRACT_SUBREG FPR128:$Rt, dsub),
6231 (CPYi64 FPR128:$Rt, (i64 1)),
6232 GPR64sp:$Rn, simm7s8:$offset)>;
6234 def : NTStore128Pat<v2i64>;
6235 def : NTStore128Pat<v4i32>;
6236 def : NTStore128Pat<v8i16>;
6237 def : NTStore128Pat<v16i8>;
6239 class NTStore64Pat<ValueType VT> :
6240 Pat<(nontemporalstore (VT FPR64:$Rt),
6241 (am_indexed7s32 GPR64sp:$Rn, simm7s4:$offset)),
6242 (STNPSi (EXTRACT_SUBREG FPR64:$Rt, ssub),
6243 (CPYi32 (SUBREG_TO_REG (i64 0), FPR64:$Rt, dsub), (i64 1)),
6244 GPR64sp:$Rn, simm7s4:$offset)>;
6246 // FIXME: Shouldn't v1f64 loads/stores be promoted to v1i64?
6247 def : NTStore64Pat<v1f64>;
6248 def : NTStore64Pat<v1i64>;
6249 def : NTStore64Pat<v2i32>;
6250 def : NTStore64Pat<v4i16>;
6251 def : NTStore64Pat<v8i8>;
6253 def : Pat<(nontemporalstore GPR64:$Rt,
6254 (am_indexed7s32 GPR64sp:$Rn, simm7s4:$offset)),
6255 (STNPWi (EXTRACT_SUBREG GPR64:$Rt, sub_32),
6256 (EXTRACT_SUBREG (UBFMXri GPR64:$Rt, 32, 63), sub_32),
6257 GPR64sp:$Rn, simm7s4:$offset)>;
6258 } // AddedComplexity=10
6259 } // Predicates = [IsLE]
6261 // Tail call return handling. These are all compiler pseudo-instructions,
6262 // so no encoding information or anything like that.
6263 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [SP] in {
6264 def TCRETURNdi : Pseudo<(outs), (ins i64imm:$dst, i32imm:$FPDiff), []>,
6265 Sched<[WriteBrReg]>;
6266 def TCRETURNri : Pseudo<(outs), (ins tcGPR64:$dst, i32imm:$FPDiff), []>,
6267 Sched<[WriteBrReg]>;
6270 def : Pat<(AArch64tcret tcGPR64:$dst, (i32 timm:$FPDiff)),
6271 (TCRETURNri tcGPR64:$dst, imm:$FPDiff)>;
6272 def : Pat<(AArch64tcret tglobaladdr:$dst, (i32 timm:$FPDiff)),
6273 (TCRETURNdi texternalsym:$dst, imm:$FPDiff)>;
6274 def : Pat<(AArch64tcret texternalsym:$dst, (i32 timm:$FPDiff)),
6275 (TCRETURNdi texternalsym:$dst, imm:$FPDiff)>;
6277 include "AArch64InstrAtomics.td"
6278 include "AArch64SVEInstrInfo.td"