contrib/llvm/lib/Target/AArch64/AArch64InstrAtomics.td

   1 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // AArch64 Atomic operand code-gen constructs.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 //===----------------------------------
  15 // Atomic fences
  16 //===----------------------------------
  17 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
  18 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
  19
  20 //===----------------------------------
  21 // Atomic loads
  22 //===----------------------------------
  23
  24 // When they're actually atomic, only one addressing mode (GPR64sp) is
  25 // supported, but when they're relaxed and anything can be used, all the
  26 // standard modes would be valid and may give efficiency gains.
  27
  28 // A atomic load operation that actually needs acquire semantics.
  29 class acquiring_load<PatFrag base>
  30   : PatFrag<(ops node:$ptr), (base node:$ptr), [{
  31   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
  32   return isAcquireOrStronger(Ordering);
  33 }]>;
  34
  35 // An atomic load operation that does not need either acquire or release
  36 // semantics.
  37 class relaxed_load<PatFrag base>
  38   : PatFrag<(ops node:$ptr), (base node:$ptr), [{
  39   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
  40   return !isAcquireOrStronger(Ordering);
  41 }]>;
  42
  43 // 8-bit loads
  44 def : Pat<(acquiring_load<atomic_load_8>  GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
  45 def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
  46                                                      ro_Wextend8:$offset)),
  47           (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
  48 def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
  49                                                      ro_Xextend8:$offset)),
  50           (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
  51 def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn,
  52                                                     uimm12s1:$offset)),
  53           (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
  54 def : Pat<(relaxed_load<atomic_load_8>
  55                (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
  56           (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
  57
  58 // 16-bit loads
  59 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
  60 def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
  61                                                        ro_Wextend16:$extend)),
  62           (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
  63 def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
  64                                                        ro_Xextend16:$extend)),
  65           (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
  66 def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn,
  67                                                       uimm12s2:$offset)),
  68           (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
  69 def : Pat<(relaxed_load<atomic_load_16>
  70                (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
  71           (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
  72
  73 // 32-bit loads
  74 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
  75 def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
  76                                                        ro_Wextend32:$extend)),
  77           (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
  78 def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
  79                                                        ro_Xextend32:$extend)),
  80           (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
  81 def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
  82                                                       uimm12s4:$offset)),
  83           (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
  84 def : Pat<(relaxed_load<atomic_load_32>
  85                (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
  86           (LDURWi GPR64sp:$Rn, simm9:$offset)>;
  87
  88 // 64-bit loads
  89 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
  90 def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
  91                                                        ro_Wextend64:$extend)),
  92           (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
  93 def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
  94                                                        ro_Xextend64:$extend)),
  95           (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
  96 def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
  97                                                       uimm12s8:$offset)),
  98           (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
  99 def : Pat<(relaxed_load<atomic_load_64>
 100                (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
 101           (LDURXi GPR64sp:$Rn, simm9:$offset)>;
 102
 103 //===----------------------------------
 104 // Atomic stores
 105 //===----------------------------------
 106
 107 // When they're actually atomic, only one addressing mode (GPR64sp) is
 108 // supported, but when they're relaxed and anything can be used, all the
 109 // standard modes would be valid and may give efficiency gains.
 110
 111 // A store operation that actually needs release semantics.
 112 class releasing_store<PatFrag base>
 113   : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
 114   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
 115   assert(Ordering != AtomicOrdering::AcquireRelease &&
 116          "unexpected store ordering");
 117   return isReleaseOrStronger(Ordering);
 118 }]>;
 119
 120 // An atomic store operation that doesn't actually need to be atomic on AArch64.
 121 class relaxed_store<PatFrag base>
 122   : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
 123   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
 124   return !isReleaseOrStronger(Ordering);
 125 }]>;
 126
 127 // 8-bit stores
 128 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
 129           (STLRB GPR32:$val, GPR64sp:$ptr)>;
 130 def : Pat<(relaxed_store<atomic_store_8>
 131                (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
 132                GPR32:$val),
 133           (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
 134 def : Pat<(relaxed_store<atomic_store_8>
 135                (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
 136                GPR32:$val),
 137           (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
 138 def : Pat<(relaxed_store<atomic_store_8>
 139                (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
 140           (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
 141 def : Pat<(relaxed_store<atomic_store_8>
 142                (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 143           (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 144
 145 // 16-bit stores
 146 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
 147           (STLRH GPR32:$val, GPR64sp:$ptr)>;
 148 def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
 149                                                          ro_Wextend16:$extend),
 150                                           GPR32:$val),
 151           (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
 152 def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
 153                                                          ro_Xextend16:$extend),
 154                                           GPR32:$val),
 155           (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
 156 def : Pat<(relaxed_store<atomic_store_16>
 157               (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
 158           (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
 159 def : Pat<(relaxed_store<atomic_store_16>
 160                (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 161           (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 162
 163 // 32-bit stores
 164 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
 165           (STLRW GPR32:$val, GPR64sp:$ptr)>;
 166 def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
 167                                                          ro_Wextend32:$extend),
 168                                           GPR32:$val),
 169           (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
 170 def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
 171                                                          ro_Xextend32:$extend),
 172                                           GPR32:$val),
 173           (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
 174 def : Pat<(relaxed_store<atomic_store_32>
 175               (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
 176           (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
 177 def : Pat<(relaxed_store<atomic_store_32>
 178                (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 179           (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 180
 181 // 64-bit stores
 182 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
 183           (STLRX GPR64:$val, GPR64sp:$ptr)>;
 184 def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
 185                                                          ro_Wextend16:$extend),
 186                                           GPR64:$val),
 187           (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
 188 def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
 189                                                          ro_Xextend16:$extend),
 190                                           GPR64:$val),
 191           (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
 192 def : Pat<(relaxed_store<atomic_store_64>
 193               (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
 194           (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
 195 def : Pat<(relaxed_store<atomic_store_64>
 196                (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
 197           (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
 198
 199 //===----------------------------------
 200 // Low-level exclusive operations
 201 //===----------------------------------
 202
 203 // Load-exclusives.
 204
 205 def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 206   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 207 }]>;
 208
 209 def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 210   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 211 }]>;
 212
 213 def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 214   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 215 }]>;
 216
 217 def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 218   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 219 }]>;
 220
 221 def : Pat<(ldxr_1 GPR64sp:$addr),
 222           (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
 223 def : Pat<(ldxr_2 GPR64sp:$addr),
 224           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 225 def : Pat<(ldxr_4 GPR64sp:$addr),
 226           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 227 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
 228
 229 def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
 230           (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
 231 def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
 232           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 233 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
 234           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 235
 236 // Load-exclusives.
 237
 238 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 239   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 240 }]>;
 241
 242 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 243   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 244 }]>;
 245
 246 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 247   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 248 }]>;
 249
 250 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 251   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 252 }]>;
 253
 254 def : Pat<(ldaxr_1 GPR64sp:$addr),
 255           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 256 def : Pat<(ldaxr_2 GPR64sp:$addr),
 257           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 258 def : Pat<(ldaxr_4 GPR64sp:$addr),
 259           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 260 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
 261
 262 def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
 263           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 264 def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
 265           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 266 def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
 267           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 268
 269 // Store-exclusives.
 270
 271 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
 272                      (int_aarch64_stxr node:$val, node:$ptr), [{
 273   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 274 }]>;
 275
 276 def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
 277                      (int_aarch64_stxr node:$val, node:$ptr), [{
 278   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 279 }]>;
 280
 281 def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
 282                      (int_aarch64_stxr node:$val, node:$ptr), [{
 283   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 284 }]>;
 285
 286 def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
 287                      (int_aarch64_stxr node:$val, node:$ptr), [{
 288   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 289 }]>;
 290
 291
 292 def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
 293           (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 294 def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
 295           (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 296 def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
 297           (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 298 def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
 299           (STXRX GPR64:$val, GPR64sp:$addr)>;
 300
 301 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
 302           (STXRB GPR32:$val, GPR64sp:$addr)>;
 303 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
 304           (STXRH GPR32:$val, GPR64sp:$addr)>;
 305 def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
 306           (STXRW GPR32:$val, GPR64sp:$addr)>;
 307
 308 def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
 309           (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 310 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
 311           (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 312 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
 313           (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 314
 315 // Store-release-exclusives.
 316
 317 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
 318                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 319   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 320 }]>;
 321
 322 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
 323                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 324   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 325 }]>;
 326
 327 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
 328                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 329   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 330 }]>;
 331
 332 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
 333                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 334   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 335 }]>;
 336
 337
 338 def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
 339           (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 340 def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
 341           (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 342 def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
 343           (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 344 def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
 345           (STLXRX GPR64:$val, GPR64sp:$addr)>;
 346
 347 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
 348           (STLXRB GPR32:$val, GPR64sp:$addr)>;
 349 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
 350           (STLXRH GPR32:$val, GPR64sp:$addr)>;
 351 def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
 352           (STLXRW GPR32:$val, GPR64sp:$addr)>;
 353
 354 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
 355           (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 356 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
 357           (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 358 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
 359           (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 360
 361
 362 // And clear exclusive.
 363
 364 def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
 365
 366 //===----------------------------------
 367 // Atomic cmpxchg for -O0
 368 //===----------------------------------
 369
 370 // The fast register allocator used during -O0 inserts spills to cover any VRegs
 371 // live across basic block boundaries. When this happens between an LDXR and an
 372 // STXR it can clear the exclusive monitor, causing all cmpxchg attempts to
 373 // fail.
 374
 375 // Unfortunately, this means we have to have an alternative (expanded
 376 // post-regalloc) path for -O0 compilations. Fortunately this path can be
 377 // significantly more naive than the standard expansion: we conservatively
 378 // assume seq_cst, strong cmpxchg and omit clrex on failure.
 379
 380 let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch",
 381     mayLoad = 1, mayStore = 1 in {
 382 def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 383                         (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 384                  Sched<[WriteAtomic]>;
 385
 386 def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 387                          (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 388                   Sched<[WriteAtomic]>;
 389
 390 def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 391                          (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 392                   Sched<[WriteAtomic]>;
 393
 394 def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
 395                          (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,
 396                   Sched<[WriteAtomic]>;
 397 }
 398
 399 let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
 400     mayLoad = 1, mayStore = 1 in
 401 def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch),
 402                           (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
 403                                GPR64:$newLo, GPR64:$newHi), []>,
 404                    Sched<[WriteAtomic]>;