contrib/llvm/lib/Target/AArch64/AArch64InstrAtomics.td

   1 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // AArch64 Atomic operand code-gen constructs.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 //===----------------------------------
  15 // Atomic fences
  16 //===----------------------------------
  17 let AddedComplexity = 15, Size = 0 in
  18 def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering),
  19                              [(atomic_fence imm:$ordering, 0)]>, Sched<[]>;
  20 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
  21 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
  22
  23 //===----------------------------------
  24 // Atomic loads
  25 //===----------------------------------
  26
  27 // When they're actually atomic, only one addressing mode (GPR64sp) is
  28 // supported, but when they're relaxed and anything can be used, all the
  29 // standard modes would be valid and may give efficiency gains.
  30
  31 // A atomic load operation that actually needs acquire semantics.
  32 class acquiring_load<PatFrag base>
  33   : PatFrag<(ops node:$ptr), (base node:$ptr), [{
  34   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
  35   return isAcquireOrStronger(Ordering);
  36 }]>;
  37
  38 // An atomic load operation that does not need either acquire or release
  39 // semantics.
  40 class relaxed_load<PatFrag base>
  41   : PatFrag<(ops node:$ptr), (base node:$ptr), [{
  42   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
  43   return !isAcquireOrStronger(Ordering);
  44 }]>;
  45
  46 // 8-bit loads
  47 def : Pat<(acquiring_load<atomic_load_8>  GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
  48 def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
  49                                                      ro_Wextend8:$offset)),
  50           (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
  51 def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
  52                                                      ro_Xextend8:$offset)),
  53           (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
  54 def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn,
  55                                                     uimm12s1:$offset)),
  56           (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
  57 def : Pat<(relaxed_load<atomic_load_8>
  58                (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
  59           (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
  60
  61 // 16-bit loads
  62 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
  63 def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
  64                                                        ro_Wextend16:$extend)),
  65           (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
  66 def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
  67                                                        ro_Xextend16:$extend)),
  68           (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
  69 def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn,
  70                                                       uimm12s2:$offset)),
  71           (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
  72 def : Pat<(relaxed_load<atomic_load_16>
  73                (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
  74           (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
  75
  76 // 32-bit loads
  77 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
  78 def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
  79                                                        ro_Wextend32:$extend)),
  80           (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
  81 def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
  82                                                        ro_Xextend32:$extend)),
  83           (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
  84 def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
  85                                                       uimm12s4:$offset)),
  86           (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
  87 def : Pat<(relaxed_load<atomic_load_32>
  88                (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
  89           (LDURWi GPR64sp:$Rn, simm9:$offset)>;
  90
  91 // 64-bit loads
  92 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
  93 def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
  94                                                        ro_Wextend64:$extend)),
  95           (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
  96 def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
  97                                                        ro_Xextend64:$extend)),
  98           (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
  99 def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
 100                                                       uimm12s8:$offset)),
 101           (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
 102 def : Pat<(relaxed_load<atomic_load_64>
 103                (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
 104           (LDURXi GPR64sp:$Rn, simm9:$offset)>;
 105
 106 //===----------------------------------
 107 // Atomic stores
 108 //===----------------------------------
 109
 110 // When they're actually atomic, only one addressing mode (GPR64sp) is
 111 // supported, but when they're relaxed and anything can be used, all the
 112 // standard modes would be valid and may give efficiency gains.
 113
 114 // A store operation that actually needs release semantics.
 115 class releasing_store<PatFrag base>
 116   : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
 117   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
 118   assert(Ordering != AtomicOrdering::AcquireRelease &&
 119          "unexpected store ordering");
 120   return isReleaseOrStronger(Ordering);
 121 }]>;
 122
 123 // An atomic store operation that doesn't actually need to be atomic on AArch64.
 124 class relaxed_store<PatFrag base>
 125   : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
 126   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
 127   return !isReleaseOrStronger(Ordering);
 128 }]>;
 129
 130 // 8-bit stores
 131 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
 132           (STLRB GPR32:$val, GPR64sp:$ptr)>;
 133 def : Pat<(relaxed_store<atomic_store_8>
 134                (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
 135                GPR32:$val),
 136           (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
 137 def : Pat<(relaxed_store<atomic_store_8>
 138                (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
 139                GPR32:$val),
 140           (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
 141 def : Pat<(relaxed_store<atomic_store_8>
 142                (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
 143           (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
 144 def : Pat<(relaxed_store<atomic_store_8>
 145                (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 146           (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 147
 148 // 16-bit stores
 149 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
 150           (STLRH GPR32:$val, GPR64sp:$ptr)>;
 151 def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
 152                                                          ro_Wextend16:$extend),
 153                                           GPR32:$val),
 154           (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
 155 def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
 156                                                          ro_Xextend16:$extend),
 157                                           GPR32:$val),
 158           (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
 159 def : Pat<(relaxed_store<atomic_store_16>
 160               (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
 161           (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
 162 def : Pat<(relaxed_store<atomic_store_16>
 163                (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 164           (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 165
 166 // 32-bit stores
 167 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
 168           (STLRW GPR32:$val, GPR64sp:$ptr)>;
 169 def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
 170                                                          ro_Wextend32:$extend),
 171                                           GPR32:$val),
 172           (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
 173 def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
 174                                                          ro_Xextend32:$extend),
 175                                           GPR32:$val),
 176           (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
 177 def : Pat<(relaxed_store<atomic_store_32>
 178               (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
 179           (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
 180 def : Pat<(relaxed_store<atomic_store_32>
 181                (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 182           (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 183
 184 // 64-bit stores
 185 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
 186           (STLRX GPR64:$val, GPR64sp:$ptr)>;
 187 def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
 188                                                          ro_Wextend16:$extend),
 189                                           GPR64:$val),
 190           (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
 191 def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
 192                                                          ro_Xextend16:$extend),
 193                                           GPR64:$val),
 194           (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
 195 def : Pat<(relaxed_store<atomic_store_64>
 196               (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
 197           (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
 198 def : Pat<(relaxed_store<atomic_store_64>
 199                (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
 200           (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
 201
 202 //===----------------------------------
 203 // Low-level exclusive operations
 204 //===----------------------------------
 205
 206 // Load-exclusives.
 207
 208 def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 209   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 210 }]>;
 211
 212 def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 213   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 214 }]>;
 215
 216 def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 217   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 218 }]>;
 219
 220 def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 221   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 222 }]>;
 223
 224 def : Pat<(ldxr_1 GPR64sp:$addr),
 225           (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
 226 def : Pat<(ldxr_2 GPR64sp:$addr),
 227           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 228 def : Pat<(ldxr_4 GPR64sp:$addr),
 229           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 230 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
 231
 232 def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
 233           (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
 234 def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
 235           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 236 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
 237           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 238
 239 // Load-exclusives.
 240
 241 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 242   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 243 }]>;
 244
 245 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 246   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 247 }]>;
 248
 249 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 250   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 251 }]>;
 252
 253 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 254   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 255 }]>;
 256
 257 def : Pat<(ldaxr_1 GPR64sp:$addr),
 258           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 259 def : Pat<(ldaxr_2 GPR64sp:$addr),
 260           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 261 def : Pat<(ldaxr_4 GPR64sp:$addr),
 262           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 263 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
 264
 265 def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
 266           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 267 def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
 268           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 269 def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
 270           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 271
 272 // Store-exclusives.
 273
 274 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
 275                      (int_aarch64_stxr node:$val, node:$ptr), [{
 276   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 277 }]>;
 278
 279 def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
 280                      (int_aarch64_stxr node:$val, node:$ptr), [{
 281   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 282 }]>;
 283
 284 def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
 285                      (int_aarch64_stxr node:$val, node:$ptr), [{
 286   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 287 }]>;
 288
 289 def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
 290                      (int_aarch64_stxr node:$val, node:$ptr), [{
 291   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 292 }]>;
 293
 294
 295 def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
 296           (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 297 def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
 298           (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 299 def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
 300           (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 301 def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
 302           (STXRX GPR64:$val, GPR64sp:$addr)>;
 303
 304 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
 305           (STXRB GPR32:$val, GPR64sp:$addr)>;
 306 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
 307           (STXRH GPR32:$val, GPR64sp:$addr)>;
 308 def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
 309           (STXRW GPR32:$val, GPR64sp:$addr)>;
 310
 311 def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
 312           (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 313 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
 314           (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 315 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
 316           (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 317
 318 // Store-release-exclusives.
 319
 320 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
 321                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 322   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 323 }]>;
 324
 325 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
 326                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 327   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 328 }]>;
 329
 330 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
 331                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 332   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 333 }]>;
 334
 335 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
 336                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 337   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 338 }]>;
 339
 340
 341 def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
 342           (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 343 def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
 344           (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 345 def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
 346           (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 347 def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
 348           (STLXRX GPR64:$val, GPR64sp:$addr)>;
 349
 350 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
 351           (STLXRB GPR32:$val, GPR64sp:$addr)>;
 352 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
 353           (STLXRH GPR32:$val, GPR64sp:$addr)>;
 354 def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
 355           (STLXRW GPR32:$val, GPR64sp:$addr)>;
 356
 357 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
 358           (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 359 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
 360           (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 361 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
 362           (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 363
 364
 365 // And clear exclusive.
 366
 367 def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
 368
 369 //===----------------------------------
 370 // Atomic cmpxchg for -O0
 371 //===----------------------------------
 372
 373 // The fast register allocator used during -O0 inserts spills to cover any VRegs
 374 // live across basic block boundaries. When this happens between an LDXR and an
 375 // STXR it can clear the exclusive monitor, causing all cmpxchg attempts to
 376 // fail.
 377
 378 // Unfortunately, this means we have to have an alternative (expanded
 379 // post-regalloc) path for -O0 compilations. Fortunately this path can be
 380 // significantly more naive than the standard expansion: we conservatively
 381 // assume seq_cst, strong cmpxchg and omit clrex on failure.
 382
 383 let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch",
 384     mayLoad = 1, mayStore = 1 in {
 385 def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 386                         (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 387                  Sched<[WriteAtomic]>;
 388
 389 def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 390                          (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 391                   Sched<[WriteAtomic]>;
 392
 393 def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 394                          (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 395                   Sched<[WriteAtomic]>;
 396
 397 def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
 398                          (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,
 399                   Sched<[WriteAtomic]>;
 400 }
 401
 402 let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
 403     mayLoad = 1, mayStore = 1 in
 404 def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch),
 405                           (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
 406                                GPR64:$newLo, GPR64:$newHi), []>,
 407                    Sched<[WriteAtomic]>;
 408
 409 // v8.1 Atomic instructions:
 410 def : Pat<(atomic_load_add_8 GPR64:$Rn, GPR32:$Rs), (LDADDALb GPR32:$Rs, GPR64sp:$Rn)>;
 411 def : Pat<(atomic_load_add_16 GPR64:$Rn, GPR32:$Rs), (LDADDALh GPR32:$Rs, GPR64sp:$Rn)>;
 412 def : Pat<(atomic_load_add_32 GPR64:$Rn, GPR32:$Rs), (LDADDALs GPR32:$Rs, GPR64sp:$Rn)>;
 413 def : Pat<(atomic_load_add_64 GPR64:$Rn, GPR64:$Rs), (LDADDALd GPR64:$Rs, GPR64sp:$Rn)>;
 414
 415 def : Pat<(atomic_load_or_8 GPR64:$Rn, GPR32:$Rs), (LDSETALb GPR32:$Rs, GPR64sp:$Rn)>;
 416 def : Pat<(atomic_load_or_16 GPR64:$Rn, GPR32:$Rs), (LDSETALh GPR32:$Rs, GPR64sp:$Rn)>;
 417 def : Pat<(atomic_load_or_32 GPR64:$Rn, GPR32:$Rs), (LDSETALs GPR32:$Rs, GPR64sp:$Rn)>;
 418 def : Pat<(atomic_load_or_64 GPR64:$Rn, GPR64:$Rs), (LDSETALd GPR64:$Rs, GPR64sp:$Rn)>;
 419
 420 def : Pat<(atomic_load_xor_8 GPR64:$Rn, GPR32:$Rs), (LDEORALb GPR32:$Rs, GPR64sp:$Rn)>;
 421 def : Pat<(atomic_load_xor_16 GPR64:$Rn, GPR32:$Rs), (LDEORALh GPR32:$Rs, GPR64sp:$Rn)>;
 422 def : Pat<(atomic_load_xor_32 GPR64:$Rn, GPR32:$Rs), (LDEORALs GPR32:$Rs, GPR64sp:$Rn)>;
 423 def : Pat<(atomic_load_xor_64 GPR64:$Rn, GPR64:$Rs), (LDEORALd GPR64:$Rs, GPR64sp:$Rn)>;
 424
 425 def : Pat<(atomic_load_max_8 GPR64:$Rn, GPR32:$Rs), (LDSMAXALb GPR32:$Rs, GPR64sp:$Rn)>;
 426 def : Pat<(atomic_load_max_16 GPR64:$Rn, GPR32:$Rs), (LDSMAXALh GPR32:$Rs, GPR64sp:$Rn)>;
 427 def : Pat<(atomic_load_max_32 GPR64:$Rn, GPR32:$Rs), (LDSMAXALs GPR32:$Rs, GPR64sp:$Rn)>;
 428 def : Pat<(atomic_load_max_64 GPR64:$Rn, GPR64:$Rs), (LDSMAXALd GPR64:$Rs, GPR64sp:$Rn)>;
 429
 430 def : Pat<(atomic_load_umax_8 GPR64:$Rn, GPR32:$Rs), (LDUMAXALb GPR32:$Rs, GPR64sp:$Rn)>;
 431 def : Pat<(atomic_load_umax_16 GPR64:$Rn, GPR32:$Rs), (LDUMAXALh GPR32:$Rs, GPR64sp:$Rn)>;
 432 def : Pat<(atomic_load_umax_32 GPR64:$Rn, GPR32:$Rs), (LDUMAXALs GPR32:$Rs, GPR64sp:$Rn)>;
 433 def : Pat<(atomic_load_umax_64 GPR64:$Rn, GPR64:$Rs), (LDUMAXALd GPR64:$Rs, GPR64sp:$Rn)>;
 434
 435 def : Pat<(atomic_load_min_8 GPR64:$Rn, GPR32:$Rs), (LDSMINALb GPR32:$Rs, GPR64sp:$Rn)>;
 436 def : Pat<(atomic_load_min_16 GPR64:$Rn, GPR32:$Rs), (LDSMINALh GPR32:$Rs, GPR64sp:$Rn)>;
 437 def : Pat<(atomic_load_min_32 GPR64:$Rn, GPR32:$Rs), (LDSMINALs GPR32:$Rs, GPR64sp:$Rn)>;
 438 def : Pat<(atomic_load_min_64 GPR64:$Rn, GPR64:$Rs), (LDSMINALd GPR64:$Rs, GPR64sp:$Rn)>;
 439
 440 def : Pat<(atomic_load_umin_8 GPR64:$Rn, GPR32:$Rs), (LDUMINALb GPR32:$Rs, GPR64sp:$Rn)>;
 441 def : Pat<(atomic_load_umin_16 GPR64:$Rn, GPR32:$Rs), (LDUMINALh GPR32:$Rs, GPR64sp:$Rn)>;
 442 def : Pat<(atomic_load_umin_32 GPR64:$Rn, GPR32:$Rs), (LDUMINALs GPR32:$Rs, GPR64sp:$Rn)>;
 443 def : Pat<(atomic_load_umin_64 GPR64:$Rn, GPR64:$Rs), (LDUMINALd GPR64:$Rs, GPR64sp:$Rn)>;
 444
 445 def : Pat<(atomic_cmp_swap_8 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALb GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
 446 def : Pat<(atomic_cmp_swap_16 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALh GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
 447 def : Pat<(atomic_cmp_swap_32 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALs GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
 448 def : Pat<(atomic_cmp_swap_64 GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALd GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>;
 449
 450 def : Pat<(atomic_swap_8 GPR64:$Rn, GPR32:$Rs), (SWPALb GPR32:$Rs, GPR64sp:$Rn)>;
 451 def : Pat<(atomic_swap_16 GPR64:$Rn, GPR32:$Rs), (SWPALh GPR32:$Rs, GPR64sp:$Rn)>;
 452 def : Pat<(atomic_swap_32 GPR64:$Rn, GPR32:$Rs), (SWPALs GPR32:$Rs, GPR64sp:$Rn)>;
 453 def : Pat<(atomic_swap_64 GPR64:$Rn, GPR64:$Rs), (SWPALd GPR64:$Rs, GPR64sp:$Rn)>;
 454
 455 def : Pat<(atomic_load_sub_8 GPR64:$Rn, GPR32:$Rs), (LDADDALb (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
 456 def : Pat<(atomic_load_sub_16 GPR64:$Rn, GPR32:$Rs), (LDADDALh (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
 457 def : Pat<(atomic_load_sub_32 GPR64:$Rn, GPR32:$Rs), (LDADDALs (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
 458 def : Pat<(atomic_load_sub_64 GPR64:$Rn, GPR64:$Rs), (LDADDALd (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>;
 459
 460 def : Pat<(atomic_load_and_8 GPR64:$Rn, GPR32:$Rs), (LDCLRALb (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
 461 def : Pat<(atomic_load_and_16 GPR64:$Rn, GPR32:$Rs), (LDCLRALh (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
 462 def : Pat<(atomic_load_and_32 GPR64:$Rn, GPR32:$Rs), (LDCLRALs (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
 463 def : Pat<(atomic_load_and_64 GPR64:$Rn, GPR64:$Rs), (LDCLRALd (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>;