1 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // AArch64 Atomic operand code-gen constructs.
12 //===----------------------------------------------------------------------===//
14 //===----------------------------------
16 //===----------------------------------
17 let AddedComplexity = 15, Size = 0 in
18 def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering),
19 [(atomic_fence imm:$ordering, 0)]>, Sched<[]>;
20 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
21 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
23 //===----------------------------------
25 //===----------------------------------
27 // When they're actually atomic, only one addressing mode (GPR64sp) is
28 // supported, but when they're relaxed and anything can be used, all the
29 // standard modes would be valid and may give efficiency gains.
31 // A atomic load operation that actually needs acquire semantics.
32 class acquiring_load<PatFrag base>
33 : PatFrag<(ops node:$ptr), (base node:$ptr), [{
34 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
35 return isAcquireOrStronger(Ordering);
38 // An atomic load operation that does not need either acquire or release
40 class relaxed_load<PatFrag base>
41 : PatFrag<(ops node:$ptr), (base node:$ptr), [{
42 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
43 return !isAcquireOrStronger(Ordering);
47 def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
48 def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
49 ro_Wextend8:$offset)),
50 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
51 def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
52 ro_Xextend8:$offset)),
53 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
54 def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn,
56 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
57 def : Pat<(relaxed_load<atomic_load_8>
58 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
59 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
62 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
63 def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
64 ro_Wextend16:$extend)),
65 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
66 def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
67 ro_Xextend16:$extend)),
68 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
69 def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn,
71 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
72 def : Pat<(relaxed_load<atomic_load_16>
73 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
74 (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
77 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
78 def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
79 ro_Wextend32:$extend)),
80 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
81 def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
82 ro_Xextend32:$extend)),
83 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
84 def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
86 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
87 def : Pat<(relaxed_load<atomic_load_32>
88 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
89 (LDURWi GPR64sp:$Rn, simm9:$offset)>;
92 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
93 def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
94 ro_Wextend64:$extend)),
95 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
96 def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
97 ro_Xextend64:$extend)),
98 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
99 def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
101 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
102 def : Pat<(relaxed_load<atomic_load_64>
103 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
104 (LDURXi GPR64sp:$Rn, simm9:$offset)>;
106 //===----------------------------------
108 //===----------------------------------
110 // When they're actually atomic, only one addressing mode (GPR64sp) is
111 // supported, but when they're relaxed and anything can be used, all the
112 // standard modes would be valid and may give efficiency gains.
114 // A store operation that actually needs release semantics.
115 class releasing_store<PatFrag base>
116 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
117 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
118 assert(Ordering != AtomicOrdering::AcquireRelease &&
119 "unexpected store ordering");
120 return isReleaseOrStronger(Ordering);
123 // An atomic store operation that doesn't actually need to be atomic on AArch64.
124 class relaxed_store<PatFrag base>
125 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
126 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
127 return !isReleaseOrStronger(Ordering);
131 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
132 (STLRB GPR32:$val, GPR64sp:$ptr)>;
133 def : Pat<(relaxed_store<atomic_store_8>
134 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
136 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
137 def : Pat<(relaxed_store<atomic_store_8>
138 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
140 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
141 def : Pat<(relaxed_store<atomic_store_8>
142 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
143 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
144 def : Pat<(relaxed_store<atomic_store_8>
145 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
146 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
149 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
150 (STLRH GPR32:$val, GPR64sp:$ptr)>;
151 def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
152 ro_Wextend16:$extend),
154 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
155 def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
156 ro_Xextend16:$extend),
158 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
159 def : Pat<(relaxed_store<atomic_store_16>
160 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
161 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
162 def : Pat<(relaxed_store<atomic_store_16>
163 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
164 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
167 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
168 (STLRW GPR32:$val, GPR64sp:$ptr)>;
169 def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
170 ro_Wextend32:$extend),
172 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
173 def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
174 ro_Xextend32:$extend),
176 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
177 def : Pat<(relaxed_store<atomic_store_32>
178 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
179 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
180 def : Pat<(relaxed_store<atomic_store_32>
181 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
182 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
185 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
186 (STLRX GPR64:$val, GPR64sp:$ptr)>;
187 def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
188 ro_Wextend16:$extend),
190 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
191 def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
192 ro_Xextend16:$extend),
194 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
195 def : Pat<(relaxed_store<atomic_store_64>
196 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
197 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
198 def : Pat<(relaxed_store<atomic_store_64>
199 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
200 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
202 //===----------------------------------
203 // Low-level exclusive operations
204 //===----------------------------------
208 def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
209 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
212 def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
213 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
216 def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
217 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
220 def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
221 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
224 def : Pat<(ldxr_1 GPR64sp:$addr),
225 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
226 def : Pat<(ldxr_2 GPR64sp:$addr),
227 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
228 def : Pat<(ldxr_4 GPR64sp:$addr),
229 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
230 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
232 def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
233 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
234 def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
235 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
236 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
237 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
241 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
242 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
245 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
246 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
249 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
250 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
253 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
254 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
257 def : Pat<(ldaxr_1 GPR64sp:$addr),
258 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
259 def : Pat<(ldaxr_2 GPR64sp:$addr),
260 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
261 def : Pat<(ldaxr_4 GPR64sp:$addr),
262 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
263 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
265 def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
266 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
267 def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
268 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
269 def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
270 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
274 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
275 (int_aarch64_stxr node:$val, node:$ptr), [{
276 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
279 def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
280 (int_aarch64_stxr node:$val, node:$ptr), [{
281 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
284 def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
285 (int_aarch64_stxr node:$val, node:$ptr), [{
286 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
289 def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
290 (int_aarch64_stxr node:$val, node:$ptr), [{
291 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
295 def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
296 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
297 def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
298 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
299 def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
300 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
301 def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
302 (STXRX GPR64:$val, GPR64sp:$addr)>;
304 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
305 (STXRB GPR32:$val, GPR64sp:$addr)>;
306 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
307 (STXRH GPR32:$val, GPR64sp:$addr)>;
308 def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
309 (STXRW GPR32:$val, GPR64sp:$addr)>;
311 def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
312 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
313 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
314 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
315 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
316 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
318 // Store-release-exclusives.
320 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
321 (int_aarch64_stlxr node:$val, node:$ptr), [{
322 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
325 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
326 (int_aarch64_stlxr node:$val, node:$ptr), [{
327 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
330 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
331 (int_aarch64_stlxr node:$val, node:$ptr), [{
332 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
335 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
336 (int_aarch64_stlxr node:$val, node:$ptr), [{
337 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
341 def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
342 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
343 def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
344 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
345 def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
346 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
347 def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
348 (STLXRX GPR64:$val, GPR64sp:$addr)>;
350 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
351 (STLXRB GPR32:$val, GPR64sp:$addr)>;
352 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
353 (STLXRH GPR32:$val, GPR64sp:$addr)>;
354 def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
355 (STLXRW GPR32:$val, GPR64sp:$addr)>;
357 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
358 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
359 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
360 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
361 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
362 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
365 // And clear exclusive.
367 def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
369 //===----------------------------------
370 // Atomic cmpxchg for -O0
371 //===----------------------------------
373 // The fast register allocator used during -O0 inserts spills to cover any VRegs
374 // live across basic block boundaries. When this happens between an LDXR and an
375 // STXR it can clear the exclusive monitor, causing all cmpxchg attempts to
378 // Unfortunately, this means we have to have an alternative (expanded
379 // post-regalloc) path for -O0 compilations. Fortunately this path can be
380 // significantly more naive than the standard expansion: we conservatively
381 // assume seq_cst, strong cmpxchg and omit clrex on failure.
383 let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch",
384 mayLoad = 1, mayStore = 1 in {
385 def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
386 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
387 Sched<[WriteAtomic]>;
389 def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
390 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
391 Sched<[WriteAtomic]>;
393 def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
394 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
395 Sched<[WriteAtomic]>;
397 def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
398 (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,
399 Sched<[WriteAtomic]>;
402 let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
403 mayLoad = 1, mayStore = 1 in
404 def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch),
405 (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
406 GPR64:$newLo, GPR64:$newHi), []>,
407 Sched<[WriteAtomic]>;
409 // v8.1 Atomic instructions:
410 def : Pat<(atomic_load_add_8 GPR64:$Rn, GPR32:$Rs), (LDADDALb GPR32:$Rs, GPR64sp:$Rn)>;
411 def : Pat<(atomic_load_add_16 GPR64:$Rn, GPR32:$Rs), (LDADDALh GPR32:$Rs, GPR64sp:$Rn)>;
412 def : Pat<(atomic_load_add_32 GPR64:$Rn, GPR32:$Rs), (LDADDALs GPR32:$Rs, GPR64sp:$Rn)>;
413 def : Pat<(atomic_load_add_64 GPR64:$Rn, GPR64:$Rs), (LDADDALd GPR64:$Rs, GPR64sp:$Rn)>;
415 def : Pat<(atomic_load_or_8 GPR64:$Rn, GPR32:$Rs), (LDSETALb GPR32:$Rs, GPR64sp:$Rn)>;
416 def : Pat<(atomic_load_or_16 GPR64:$Rn, GPR32:$Rs), (LDSETALh GPR32:$Rs, GPR64sp:$Rn)>;
417 def : Pat<(atomic_load_or_32 GPR64:$Rn, GPR32:$Rs), (LDSETALs GPR32:$Rs, GPR64sp:$Rn)>;
418 def : Pat<(atomic_load_or_64 GPR64:$Rn, GPR64:$Rs), (LDSETALd GPR64:$Rs, GPR64sp:$Rn)>;
420 def : Pat<(atomic_load_xor_8 GPR64:$Rn, GPR32:$Rs), (LDEORALb GPR32:$Rs, GPR64sp:$Rn)>;
421 def : Pat<(atomic_load_xor_16 GPR64:$Rn, GPR32:$Rs), (LDEORALh GPR32:$Rs, GPR64sp:$Rn)>;
422 def : Pat<(atomic_load_xor_32 GPR64:$Rn, GPR32:$Rs), (LDEORALs GPR32:$Rs, GPR64sp:$Rn)>;
423 def : Pat<(atomic_load_xor_64 GPR64:$Rn, GPR64:$Rs), (LDEORALd GPR64:$Rs, GPR64sp:$Rn)>;
425 def : Pat<(atomic_load_max_8 GPR64:$Rn, GPR32:$Rs), (LDSMAXALb GPR32:$Rs, GPR64sp:$Rn)>;
426 def : Pat<(atomic_load_max_16 GPR64:$Rn, GPR32:$Rs), (LDSMAXALh GPR32:$Rs, GPR64sp:$Rn)>;
427 def : Pat<(atomic_load_max_32 GPR64:$Rn, GPR32:$Rs), (LDSMAXALs GPR32:$Rs, GPR64sp:$Rn)>;
428 def : Pat<(atomic_load_max_64 GPR64:$Rn, GPR64:$Rs), (LDSMAXALd GPR64:$Rs, GPR64sp:$Rn)>;
430 def : Pat<(atomic_load_umax_8 GPR64:$Rn, GPR32:$Rs), (LDUMAXALb GPR32:$Rs, GPR64sp:$Rn)>;
431 def : Pat<(atomic_load_umax_16 GPR64:$Rn, GPR32:$Rs), (LDUMAXALh GPR32:$Rs, GPR64sp:$Rn)>;
432 def : Pat<(atomic_load_umax_32 GPR64:$Rn, GPR32:$Rs), (LDUMAXALs GPR32:$Rs, GPR64sp:$Rn)>;
433 def : Pat<(atomic_load_umax_64 GPR64:$Rn, GPR64:$Rs), (LDUMAXALd GPR64:$Rs, GPR64sp:$Rn)>;
435 def : Pat<(atomic_load_min_8 GPR64:$Rn, GPR32:$Rs), (LDSMINALb GPR32:$Rs, GPR64sp:$Rn)>;
436 def : Pat<(atomic_load_min_16 GPR64:$Rn, GPR32:$Rs), (LDSMINALh GPR32:$Rs, GPR64sp:$Rn)>;
437 def : Pat<(atomic_load_min_32 GPR64:$Rn, GPR32:$Rs), (LDSMINALs GPR32:$Rs, GPR64sp:$Rn)>;
438 def : Pat<(atomic_load_min_64 GPR64:$Rn, GPR64:$Rs), (LDSMINALd GPR64:$Rs, GPR64sp:$Rn)>;
440 def : Pat<(atomic_load_umin_8 GPR64:$Rn, GPR32:$Rs), (LDUMINALb GPR32:$Rs, GPR64sp:$Rn)>;
441 def : Pat<(atomic_load_umin_16 GPR64:$Rn, GPR32:$Rs), (LDUMINALh GPR32:$Rs, GPR64sp:$Rn)>;
442 def : Pat<(atomic_load_umin_32 GPR64:$Rn, GPR32:$Rs), (LDUMINALs GPR32:$Rs, GPR64sp:$Rn)>;
443 def : Pat<(atomic_load_umin_64 GPR64:$Rn, GPR64:$Rs), (LDUMINALd GPR64:$Rs, GPR64sp:$Rn)>;
445 def : Pat<(atomic_cmp_swap_8 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALb GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
446 def : Pat<(atomic_cmp_swap_16 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALh GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
447 def : Pat<(atomic_cmp_swap_32 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALs GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
448 def : Pat<(atomic_cmp_swap_64 GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALd GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>;
450 def : Pat<(atomic_swap_8 GPR64:$Rn, GPR32:$Rs), (SWPALb GPR32:$Rs, GPR64sp:$Rn)>;
451 def : Pat<(atomic_swap_16 GPR64:$Rn, GPR32:$Rs), (SWPALh GPR32:$Rs, GPR64sp:$Rn)>;
452 def : Pat<(atomic_swap_32 GPR64:$Rn, GPR32:$Rs), (SWPALs GPR32:$Rs, GPR64sp:$Rn)>;
453 def : Pat<(atomic_swap_64 GPR64:$Rn, GPR64:$Rs), (SWPALd GPR64:$Rs, GPR64sp:$Rn)>;
455 def : Pat<(atomic_load_sub_8 GPR64:$Rn, GPR32:$Rs), (LDADDALb (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
456 def : Pat<(atomic_load_sub_16 GPR64:$Rn, GPR32:$Rs), (LDADDALh (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
457 def : Pat<(atomic_load_sub_32 GPR64:$Rn, GPR32:$Rs), (LDADDALs (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
458 def : Pat<(atomic_load_sub_64 GPR64:$Rn, GPR64:$Rs), (LDADDALd (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>;
460 def : Pat<(atomic_load_and_8 GPR64:$Rn, GPR32:$Rs), (LDCLRALb (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
461 def : Pat<(atomic_load_and_16 GPR64:$Rn, GPR32:$Rs), (LDCLRALh (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
462 def : Pat<(atomic_load_and_32 GPR64:$Rn, GPR32:$Rs), (LDCLRALs (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
463 def : Pat<(atomic_load_and_64 GPR64:$Rn, GPR64:$Rs), (LDCLRALd (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>;