contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp

   1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the code for emitting atomic operations.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "CGCall.h"
  14 #include "CGRecordLayout.h"
  15 #include "CodeGenFunction.h"
  16 #include "CodeGenModule.h"
  17 #include "TargetInfo.h"
  18 #include "clang/AST/ASTContext.h"
  19 #include "clang/CodeGen/CGFunctionInfo.h"
  20 #include "clang/Frontend/FrontendDiagnostic.h"
  21 #include "llvm/ADT/DenseMap.h"
  22 #include "llvm/IR/DataLayout.h"
  23 #include "llvm/IR/Intrinsics.h"
  24 #include "llvm/IR/Operator.h"
  25
  26 using namespace clang;
  27 using namespace CodeGen;
  28
  29 namespace {
  30   class AtomicInfo {
  31     CodeGenFunction &CGF;
  32     QualType AtomicTy;
  33     QualType ValueTy;
  34     uint64_t AtomicSizeInBits;
  35     uint64_t ValueSizeInBits;
  36     CharUnits AtomicAlign;
  37     CharUnits ValueAlign;
  38     TypeEvaluationKind EvaluationKind;
  39     bool UseLibcall;
  40     LValue LVal;
  41     CGBitFieldInfo BFI;
  42   public:
  43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
  44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
  45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
  46       assert(!lvalue.isGlobalReg());
  47       ASTContext &C = CGF.getContext();
  48       if (lvalue.isSimple()) {
  49         AtomicTy = lvalue.getType();
  50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
  51           ValueTy = ATy->getValueType();
  52         else
  53           ValueTy = AtomicTy;
  54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
  55
  56         uint64_t ValueAlignInBits;
  57         uint64_t AtomicAlignInBits;
  58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
  59         ValueSizeInBits = ValueTI.Width;
  60         ValueAlignInBits = ValueTI.Align;
  61
  62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
  63         AtomicSizeInBits = AtomicTI.Width;
  64         AtomicAlignInBits = AtomicTI.Align;
  65
  66         assert(ValueSizeInBits <= AtomicSizeInBits);
  67         assert(ValueAlignInBits <= AtomicAlignInBits);
  68
  69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
  70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
  71         if (lvalue.getAlignment().isZero())
  72           lvalue.setAlignment(AtomicAlign);
  73
  74         LVal = lvalue;
  75       } else if (lvalue.isBitField()) {
  76         ValueTy = lvalue.getType();
  77         ValueSizeInBits = C.getTypeSize(ValueTy);
  78         auto &OrigBFI = lvalue.getBitFieldInfo();
  79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
  80         AtomicSizeInBits = C.toBits(
  81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
  82                 .alignTo(lvalue.getAlignment()));
  83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
  84         auto OffsetInChars =
  85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
  86             lvalue.getAlignment();
  87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
  88             CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
  89         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
  90             VoidPtrAddr,
  91             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
  92             "atomic_bitfield_base");
  93         BFI = OrigBFI;
  94         BFI.Offset = Offset;
  95         BFI.StorageSize = AtomicSizeInBits;
  96         BFI.StorageOffset += OffsetInChars;
  97         LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
  98                                     BFI, lvalue.getType(), lvalue.getBaseInfo(),
  99                                     lvalue.getTBAAInfo());
 100         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
 101         if (AtomicTy.isNull()) {
 102           llvm::APInt Size(
 103               /*numBits=*/32,
 104               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
 105           AtomicTy =
 106               C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
 107                                      /*IndexTypeQuals=*/0);
 108         }
 109         AtomicAlign = ValueAlign = lvalue.getAlignment();
 110       } else if (lvalue.isVectorElt()) {
 111         ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
 112         ValueSizeInBits = C.getTypeSize(ValueTy);
 113         AtomicTy = lvalue.getType();
 114         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 115         AtomicAlign = ValueAlign = lvalue.getAlignment();
 116         LVal = lvalue;
 117       } else {
 118         assert(lvalue.isExtVectorElt());
 119         ValueTy = lvalue.getType();
 120         ValueSizeInBits = C.getTypeSize(ValueTy);
 121         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
 122             lvalue.getType(), cast<llvm::FixedVectorType>(
 123                                   lvalue.getExtVectorAddress().getElementType())
 124                                   ->getNumElements());
 125         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 126         AtomicAlign = ValueAlign = lvalue.getAlignment();
 127         LVal = lvalue;
 128       }
 129       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
 130           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
 131     }
 132
 133     QualType getAtomicType() const { return AtomicTy; }
 134     QualType getValueType() const { return ValueTy; }
 135     CharUnits getAtomicAlignment() const { return AtomicAlign; }
 136     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
 137     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
 138     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
 139     bool shouldUseLibcall() const { return UseLibcall; }
 140     const LValue &getAtomicLValue() const { return LVal; }
 141     llvm::Value *getAtomicPointer() const {
 142       if (LVal.isSimple())
 143         return LVal.getPointer(CGF);
 144       else if (LVal.isBitField())
 145         return LVal.getBitFieldPointer();
 146       else if (LVal.isVectorElt())
 147         return LVal.getVectorPointer();
 148       assert(LVal.isExtVectorElt());
 149       return LVal.getExtVectorPointer();
 150     }
 151     Address getAtomicAddress() const {
 152       return Address(getAtomicPointer(), getAtomicAlignment());
 153     }
 154
 155     Address getAtomicAddressAsAtomicIntPointer() const {
 156       return emitCastToAtomicIntPointer(getAtomicAddress());
 157     }
 158
 159     /// Is the atomic size larger than the underlying value type?
 160     ///
 161     /// Note that the absence of padding does not mean that atomic
 162     /// objects are completely interchangeable with non-atomic
 163     /// objects: we might have promoted the alignment of a type
 164     /// without making it bigger.
 165     bool hasPadding() const {
 166       return (ValueSizeInBits != AtomicSizeInBits);
 167     }
 168
 169     bool emitMemSetZeroIfNecessary() const;
 170
 171     llvm::Value *getAtomicSizeValue() const {
 172       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
 173       return CGF.CGM.getSize(size);
 174     }
 175
 176     /// Cast the given pointer to an integer pointer suitable for atomic
 177     /// operations if the source.
 178     Address emitCastToAtomicIntPointer(Address Addr) const;
 179
 180     /// If Addr is compatible with the iN that will be used for an atomic
 181     /// operation, bitcast it. Otherwise, create a temporary that is suitable
 182     /// and copy the value across.
 183     Address convertToAtomicIntPointer(Address Addr) const;
 184
 185     /// Turn an atomic-layout object into an r-value.
 186     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
 187                                      SourceLocation loc, bool AsValue) const;
 188
 189     /// Converts a rvalue to integer value.
 190     llvm::Value *convertRValueToInt(RValue RVal) const;
 191
 192     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
 193                                      AggValueSlot ResultSlot,
 194                                      SourceLocation Loc, bool AsValue) const;
 195
 196     /// Copy an atomic r-value into atomic-layout memory.
 197     void emitCopyIntoMemory(RValue rvalue) const;
 198
 199     /// Project an l-value down to the value field.
 200     LValue projectValue() const {
 201       assert(LVal.isSimple());
 202       Address addr = getAtomicAddress();
 203       if (hasPadding())
 204         addr = CGF.Builder.CreateStructGEP(addr, 0);
 205
 206       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
 207                               LVal.getBaseInfo(), LVal.getTBAAInfo());
 208     }
 209
 210     /// Emits atomic load.
 211     /// \returns Loaded value.
 212     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
 213                           bool AsValue, llvm::AtomicOrdering AO,
 214                           bool IsVolatile);
 215
 216     /// Emits atomic compare-and-exchange sequence.
 217     /// \param Expected Expected value.
 218     /// \param Desired Desired value.
 219     /// \param Success Atomic ordering for success operation.
 220     /// \param Failure Atomic ordering for failed operation.
 221     /// \param IsWeak true if atomic operation is weak, false otherwise.
 222     /// \returns Pair of values: previous value from storage (value type) and
 223     /// boolean flag (i1 type) with true if success and false otherwise.
 224     std::pair<RValue, llvm::Value *>
 225     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
 226                               llvm::AtomicOrdering Success =
 227                                   llvm::AtomicOrdering::SequentiallyConsistent,
 228                               llvm::AtomicOrdering Failure =
 229                                   llvm::AtomicOrdering::SequentiallyConsistent,
 230                               bool IsWeak = false);
 231
 232     /// Emits atomic update.
 233     /// \param AO Atomic ordering.
 234     /// \param UpdateOp Update operation for the current lvalue.
 235     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
 236                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
 237                           bool IsVolatile);
 238     /// Emits atomic update.
 239     /// \param AO Atomic ordering.
 240     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
 241                           bool IsVolatile);
 242
 243     /// Materialize an atomic r-value in atomic-layout memory.
 244     Address materializeRValue(RValue rvalue) const;
 245
 246     /// Creates temp alloca for intermediate operations on atomic value.
 247     Address CreateTempAlloca() const;
 248   private:
 249     bool requiresMemSetZero(llvm::Type *type) const;
 250
 251
 252     /// Emits atomic load as a libcall.
 253     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
 254                                llvm::AtomicOrdering AO, bool IsVolatile);
 255     /// Emits atomic load as LLVM instruction.
 256     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
 257     /// Emits atomic compare-and-exchange op as a libcall.
 258     llvm::Value *EmitAtomicCompareExchangeLibcall(
 259         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
 260         llvm::AtomicOrdering Success =
 261             llvm::AtomicOrdering::SequentiallyConsistent,
 262         llvm::AtomicOrdering Failure =
 263             llvm::AtomicOrdering::SequentiallyConsistent);
 264     /// Emits atomic compare-and-exchange op as LLVM instruction.
 265     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
 266         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
 267         llvm::AtomicOrdering Success =
 268             llvm::AtomicOrdering::SequentiallyConsistent,
 269         llvm::AtomicOrdering Failure =
 270             llvm::AtomicOrdering::SequentiallyConsistent,
 271         bool IsWeak = false);
 272     /// Emit atomic update as libcalls.
 273     void
 274     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
 275                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 276                             bool IsVolatile);
 277     /// Emit atomic update as LLVM instructions.
 278     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
 279                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 280                             bool IsVolatile);
 281     /// Emit atomic update as libcalls.
 282     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
 283                                  bool IsVolatile);
 284     /// Emit atomic update as LLVM instructions.
 285     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
 286                             bool IsVolatile);
 287   };
 288 }
 289
 290 Address AtomicInfo::CreateTempAlloca() const {
 291   Address TempAlloca = CGF.CreateMemTemp(
 292       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
 293                                                                 : AtomicTy,
 294       getAtomicAlignment(),
 295       "atomic-temp");
 296   // Cast to pointer to value type for bitfields.
 297   if (LVal.isBitField())
 298     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
 299         TempAlloca, getAtomicAddress().getType());
 300   return TempAlloca;
 301 }
 302
 303 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
 304                                 StringRef fnName,
 305                                 QualType resultType,
 306                                 CallArgList &args) {
 307   const CGFunctionInfo &fnInfo =
 308     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
 309   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
 310   llvm::AttrBuilder fnAttrB;
 311   fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
 312   fnAttrB.addAttribute(llvm::Attribute::WillReturn);
 313   llvm::AttributeList fnAttrs = llvm::AttributeList::get(
 314       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
 315
 316   llvm::FunctionCallee fn =
 317       CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
 318   auto callee = CGCallee::forDirect(fn);
 319   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
 320 }
 321
 322 /// Does a store of the given IR type modify the full expected width?
 323 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
 324                            uint64_t expectedSize) {
 325   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
 326 }
 327
 328 /// Does the atomic type require memsetting to zero before initialization?
 329 ///
 330 /// The IR type is provided as a way of making certain queries faster.
 331 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
 332   // If the atomic type has size padding, we definitely need a memset.
 333   if (hasPadding()) return true;
 334
 335   // Otherwise, do some simple heuristics to try to avoid it:
 336   switch (getEvaluationKind()) {
 337   // For scalars and complexes, check whether the store size of the
 338   // type uses the full size.
 339   case TEK_Scalar:
 340     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
 341   case TEK_Complex:
 342     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
 343                            AtomicSizeInBits / 2);
 344
 345   // Padding in structs has an undefined bit pattern.  User beware.
 346   case TEK_Aggregate:
 347     return false;
 348   }
 349   llvm_unreachable("bad evaluation kind");
 350 }
 351
 352 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
 353   assert(LVal.isSimple());
 354   llvm::Value *addr = LVal.getPointer(CGF);
 355   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
 356     return false;
 357
 358   CGF.Builder.CreateMemSet(
 359       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
 360       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
 361       LVal.getAlignment().getAsAlign());
 362   return true;
 363 }
 364
 365 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
 366                               Address Dest, Address Ptr,
 367                               Address Val1, Address Val2,
 368                               uint64_t Size,
 369                               llvm::AtomicOrdering SuccessOrder,
 370                               llvm::AtomicOrdering FailureOrder,
 371                               llvm::SyncScope::ID Scope) {
 372   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
 373   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
 374   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
 375
 376   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
 377       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
 378       Scope);
 379   Pair->setVolatile(E->isVolatile());
 380   Pair->setWeak(IsWeak);
 381
 382   // Cmp holds the result of the compare-exchange operation: true on success,
 383   // false on failure.
 384   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
 385   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
 386
 387   // This basic block is used to hold the store instruction if the operation
 388   // failed.
 389   llvm::BasicBlock *StoreExpectedBB =
 390       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
 391
 392   // This basic block is the exit point of the operation, we should end up
 393   // here regardless of whether or not the operation succeeded.
 394   llvm::BasicBlock *ContinueBB =
 395       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 396
 397   // Update Expected if Expected isn't equal to Old, otherwise branch to the
 398   // exit point.
 399   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
 400
 401   CGF.Builder.SetInsertPoint(StoreExpectedBB);
 402   // Update the memory at Expected with Old's value.
 403   CGF.Builder.CreateStore(Old, Val1);
 404   // Finally, branch to the exit point.
 405   CGF.Builder.CreateBr(ContinueBB);
 406
 407   CGF.Builder.SetInsertPoint(ContinueBB);
 408   // Update the memory at Dest with Cmp's value.
 409   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
 410 }
 411
 412 /// Given an ordering required on success, emit all possible cmpxchg
 413 /// instructions to cope with the provided (but possibly only dynamically known)
 414 /// FailureOrder.
 415 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
 416                                         bool IsWeak, Address Dest, Address Ptr,
 417                                         Address Val1, Address Val2,
 418                                         llvm::Value *FailureOrderVal,
 419                                         uint64_t Size,
 420                                         llvm::AtomicOrdering SuccessOrder,
 421                                         llvm::SyncScope::ID Scope) {
 422   llvm::AtomicOrdering FailureOrder;
 423   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
 424     auto FOS = FO->getSExtValue();
 425     if (!llvm::isValidAtomicOrderingCABI(FOS))
 426       FailureOrder = llvm::AtomicOrdering::Monotonic;
 427     else
 428       switch ((llvm::AtomicOrderingCABI)FOS) {
 429       case llvm::AtomicOrderingCABI::relaxed:
 430       // 31.7.2.18: "The failure argument shall not be memory_order_release
 431       // nor memory_order_acq_rel". Fallback to monotonic.
 432       case llvm::AtomicOrderingCABI::release:
 433       case llvm::AtomicOrderingCABI::acq_rel:
 434         FailureOrder = llvm::AtomicOrdering::Monotonic;
 435         break;
 436       case llvm::AtomicOrderingCABI::consume:
 437       case llvm::AtomicOrderingCABI::acquire:
 438         FailureOrder = llvm::AtomicOrdering::Acquire;
 439         break;
 440       case llvm::AtomicOrderingCABI::seq_cst:
 441         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
 442         break;
 443       }
 444     // Prior to c++17, "the failure argument shall be no stronger than the
 445     // success argument". This condition has been lifted and the only
 446     // precondition is 31.7.2.18. Effectively treat this as a DR and skip
 447     // language version checks.
 448     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
 449                       FailureOrder, Scope);
 450     return;
 451   }
 452
 453   // Create all the relevant BB's
 454   auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
 455   auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
 456   auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
 457   auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
 458
 459   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
 460   // doesn't matter unless someone is crazy enough to use something that
 461   // doesn't fold to a constant for the ordering.
 462   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
 463   // Implemented as acquire, since it's the closest in LLVM.
 464   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
 465               AcquireBB);
 466   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
 467               AcquireBB);
 468   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
 469               SeqCstBB);
 470
 471   // Emit all the different atomics
 472   CGF.Builder.SetInsertPoint(MonotonicBB);
 473   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 474                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
 475   CGF.Builder.CreateBr(ContBB);
 476
 477   CGF.Builder.SetInsertPoint(AcquireBB);
 478   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
 479                     llvm::AtomicOrdering::Acquire, Scope);
 480   CGF.Builder.CreateBr(ContBB);
 481
 482   CGF.Builder.SetInsertPoint(SeqCstBB);
 483   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
 484                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
 485   CGF.Builder.CreateBr(ContBB);
 486
 487   CGF.Builder.SetInsertPoint(ContBB);
 488 }
 489
 490 /// Duplicate the atomic min/max operation in conventional IR for the builtin
 491 /// variants that return the new rather than the original value.
 492 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
 493                                          AtomicExpr::AtomicOp Op,
 494                                          bool IsSigned,
 495                                          llvm::Value *OldVal,
 496                                          llvm::Value *RHS) {
 497   llvm::CmpInst::Predicate Pred;
 498   switch (Op) {
 499   default:
 500     llvm_unreachable("Unexpected min/max operation");
 501   case AtomicExpr::AO__atomic_max_fetch:
 502     Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
 503     break;
 504   case AtomicExpr::AO__atomic_min_fetch:
 505     Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
 506     break;
 507   }
 508   llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
 509   return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
 510 }
 511
 512 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
 513                          Address Ptr, Address Val1, Address Val2,
 514                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 515                          uint64_t Size, llvm::AtomicOrdering Order,
 516                          llvm::SyncScope::ID Scope) {
 517   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
 518   bool PostOpMinMax = false;
 519   unsigned PostOp = 0;
 520
 521   switch (E->getOp()) {
 522   case AtomicExpr::AO__c11_atomic_init:
 523   case AtomicExpr::AO__opencl_atomic_init:
 524     llvm_unreachable("Already handled!");
 525
 526   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 527   case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
 528   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
 529     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 530                                 FailureOrder, Size, Order, Scope);
 531     return;
 532   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 533   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
 534   case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
 535     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 536                                 FailureOrder, Size, Order, Scope);
 537     return;
 538   case AtomicExpr::AO__atomic_compare_exchange:
 539   case AtomicExpr::AO__atomic_compare_exchange_n: {
 540     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
 541       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
 542                                   Val1, Val2, FailureOrder, Size, Order, Scope);
 543     } else {
 544       // Create all the relevant BB's
 545       llvm::BasicBlock *StrongBB =
 546           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
 547       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
 548       llvm::BasicBlock *ContBB =
 549           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 550
 551       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
 552       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
 553
 554       CGF.Builder.SetInsertPoint(StrongBB);
 555       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 556                                   FailureOrder, Size, Order, Scope);
 557       CGF.Builder.CreateBr(ContBB);
 558
 559       CGF.Builder.SetInsertPoint(WeakBB);
 560       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 561                                   FailureOrder, Size, Order, Scope);
 562       CGF.Builder.CreateBr(ContBB);
 563
 564       CGF.Builder.SetInsertPoint(ContBB);
 565     }
 566     return;
 567   }
 568   case AtomicExpr::AO__c11_atomic_load:
 569   case AtomicExpr::AO__opencl_atomic_load:
 570   case AtomicExpr::AO__hip_atomic_load:
 571   case AtomicExpr::AO__atomic_load_n:
 572   case AtomicExpr::AO__atomic_load: {
 573     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
 574     Load->setAtomic(Order, Scope);
 575     Load->setVolatile(E->isVolatile());
 576     CGF.Builder.CreateStore(Load, Dest);
 577     return;
 578   }
 579
 580   case AtomicExpr::AO__c11_atomic_store:
 581   case AtomicExpr::AO__opencl_atomic_store:
 582   case AtomicExpr::AO__hip_atomic_store:
 583   case AtomicExpr::AO__atomic_store:
 584   case AtomicExpr::AO__atomic_store_n: {
 585     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 586     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
 587     Store->setAtomic(Order, Scope);
 588     Store->setVolatile(E->isVolatile());
 589     return;
 590   }
 591
 592   case AtomicExpr::AO__c11_atomic_exchange:
 593   case AtomicExpr::AO__hip_atomic_exchange:
 594   case AtomicExpr::AO__opencl_atomic_exchange:
 595   case AtomicExpr::AO__atomic_exchange_n:
 596   case AtomicExpr::AO__atomic_exchange:
 597     Op = llvm::AtomicRMWInst::Xchg;
 598     break;
 599
 600   case AtomicExpr::AO__atomic_add_fetch:
 601     PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
 602                                                  : llvm::Instruction::Add;
 603     LLVM_FALLTHROUGH;
 604   case AtomicExpr::AO__c11_atomic_fetch_add:
 605   case AtomicExpr::AO__hip_atomic_fetch_add:
 606   case AtomicExpr::AO__opencl_atomic_fetch_add:
 607   case AtomicExpr::AO__atomic_fetch_add:
 608     Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
 609                                              : llvm::AtomicRMWInst::Add;
 610     break;
 611
 612   case AtomicExpr::AO__atomic_sub_fetch:
 613     PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
 614                                                  : llvm::Instruction::Sub;
 615     LLVM_FALLTHROUGH;
 616   case AtomicExpr::AO__c11_atomic_fetch_sub:
 617   case AtomicExpr::AO__opencl_atomic_fetch_sub:
 618   case AtomicExpr::AO__atomic_fetch_sub:
 619     Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
 620                                              : llvm::AtomicRMWInst::Sub;
 621     break;
 622
 623   case AtomicExpr::AO__atomic_min_fetch:
 624     PostOpMinMax = true;
 625     LLVM_FALLTHROUGH;
 626   case AtomicExpr::AO__c11_atomic_fetch_min:
 627   case AtomicExpr::AO__hip_atomic_fetch_min:
 628   case AtomicExpr::AO__opencl_atomic_fetch_min:
 629   case AtomicExpr::AO__atomic_fetch_min:
 630     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
 631                                                   : llvm::AtomicRMWInst::UMin;
 632     break;
 633
 634   case AtomicExpr::AO__atomic_max_fetch:
 635     PostOpMinMax = true;
 636     LLVM_FALLTHROUGH;
 637   case AtomicExpr::AO__c11_atomic_fetch_max:
 638   case AtomicExpr::AO__hip_atomic_fetch_max:
 639   case AtomicExpr::AO__opencl_atomic_fetch_max:
 640   case AtomicExpr::AO__atomic_fetch_max:
 641     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
 642                                                   : llvm::AtomicRMWInst::UMax;
 643     break;
 644
 645   case AtomicExpr::AO__atomic_and_fetch:
 646     PostOp = llvm::Instruction::And;
 647     LLVM_FALLTHROUGH;
 648   case AtomicExpr::AO__c11_atomic_fetch_and:
 649   case AtomicExpr::AO__hip_atomic_fetch_and:
 650   case AtomicExpr::AO__opencl_atomic_fetch_and:
 651   case AtomicExpr::AO__atomic_fetch_and:
 652     Op = llvm::AtomicRMWInst::And;
 653     break;
 654
 655   case AtomicExpr::AO__atomic_or_fetch:
 656     PostOp = llvm::Instruction::Or;
 657     LLVM_FALLTHROUGH;
 658   case AtomicExpr::AO__c11_atomic_fetch_or:
 659   case AtomicExpr::AO__hip_atomic_fetch_or:
 660   case AtomicExpr::AO__opencl_atomic_fetch_or:
 661   case AtomicExpr::AO__atomic_fetch_or:
 662     Op = llvm::AtomicRMWInst::Or;
 663     break;
 664
 665   case AtomicExpr::AO__atomic_xor_fetch:
 666     PostOp = llvm::Instruction::Xor;
 667     LLVM_FALLTHROUGH;
 668   case AtomicExpr::AO__c11_atomic_fetch_xor:
 669   case AtomicExpr::AO__hip_atomic_fetch_xor:
 670   case AtomicExpr::AO__opencl_atomic_fetch_xor:
 671   case AtomicExpr::AO__atomic_fetch_xor:
 672     Op = llvm::AtomicRMWInst::Xor;
 673     break;
 674
 675   case AtomicExpr::AO__atomic_nand_fetch:
 676     PostOp = llvm::Instruction::And; // the NOT is special cased below
 677     LLVM_FALLTHROUGH;
 678   case AtomicExpr::AO__c11_atomic_fetch_nand:
 679   case AtomicExpr::AO__atomic_fetch_nand:
 680     Op = llvm::AtomicRMWInst::Nand;
 681     break;
 682   }
 683
 684   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 685   llvm::AtomicRMWInst *RMWI =
 686       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
 687   RMWI->setVolatile(E->isVolatile());
 688
 689   // For __atomic_*_fetch operations, perform the operation again to
 690   // determine the value which was written.
 691   llvm::Value *Result = RMWI;
 692   if (PostOpMinMax)
 693     Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
 694                                   E->getValueType()->isSignedIntegerType(),
 695                                   RMWI, LoadVal1);
 696   else if (PostOp)
 697     Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
 698                                      LoadVal1);
 699   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
 700     Result = CGF.Builder.CreateNot(Result);
 701   CGF.Builder.CreateStore(Result, Dest);
 702 }
 703
 704 // This function emits any expression (scalar, complex, or aggregate)
 705 // into a temporary alloca.
 706 static Address
 707 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
 708   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
 709   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
 710                        /*Init*/ true);
 711   return DeclPtr;
 712 }
 713
 714 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
 715                          Address Ptr, Address Val1, Address Val2,
 716                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 717                          uint64_t Size, llvm::AtomicOrdering Order,
 718                          llvm::Value *Scope) {
 719   auto ScopeModel = Expr->getScopeModel();
 720
 721   // LLVM atomic instructions always have synch scope. If clang atomic
 722   // expression has no scope operand, use default LLVM synch scope.
 723   if (!ScopeModel) {
 724     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
 725                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
 726     return;
 727   }
 728
 729   // Handle constant scope.
 730   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
 731     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
 732         CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
 733         Order, CGF.CGM.getLLVMContext());
 734     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
 735                  Order, SCID);
 736     return;
 737   }
 738
 739   // Handle non-constant scope.
 740   auto &Builder = CGF.Builder;
 741   auto Scopes = ScopeModel->getRuntimeValues();
 742   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
 743   for (auto S : Scopes)
 744     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
 745
 746   llvm::BasicBlock *ContBB =
 747       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
 748
 749   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
 750   // If unsupported synch scope is encountered at run time, assume a fallback
 751   // synch scope value.
 752   auto FallBack = ScopeModel->getFallBackValue();
 753   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
 754   for (auto S : Scopes) {
 755     auto *B = BB[S];
 756     if (S != FallBack)
 757       SI->addCase(Builder.getInt32(S), B);
 758
 759     Builder.SetInsertPoint(B);
 760     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
 761                  Order,
 762                  CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
 763                                                          ScopeModel->map(S),
 764                                                          Order,
 765                                                          CGF.getLLVMContext()));
 766     Builder.CreateBr(ContBB);
 767   }
 768
 769   Builder.SetInsertPoint(ContBB);
 770 }
 771
 772 static void
 773 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
 774                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
 775                   SourceLocation Loc, CharUnits SizeInChars) {
 776   if (UseOptimizedLibcall) {
 777     // Load value and pass it to the function directly.
 778     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
 779     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
 780     ValTy =
 781         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
 782     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
 783                                                 SizeInBits)->getPointerTo();
 784     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
 785     Val = CGF.EmitLoadOfScalar(Ptr, false,
 786                                CGF.getContext().getPointerType(ValTy),
 787                                Loc);
 788     // Coerce the value into an appropriately sized integer type.
 789     Args.add(RValue::get(Val), ValTy);
 790   } else {
 791     // Non-optimized functions always take a reference.
 792     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
 793                          CGF.getContext().VoidPtrTy);
 794   }
 795 }
 796
 797 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
 798   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
 799   QualType MemTy = AtomicTy;
 800   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
 801     MemTy = AT->getValueType();
 802   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
 803
 804   Address Val1 = Address::invalid();
 805   Address Val2 = Address::invalid();
 806   Address Dest = Address::invalid();
 807   Address Ptr = EmitPointerWithAlignment(E->getPtr());
 808
 809   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
 810       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
 811     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
 812     EmitAtomicInit(E->getVal1(), lvalue);
 813     return RValue::get(nullptr);
 814   }
 815
 816   auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
 817   uint64_t Size = TInfo.Width.getQuantity();
 818   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
 819
 820   bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
 821   bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
 822   bool UseLibcall = Misaligned | Oversized;
 823   bool ShouldCastToIntPtrTy = true;
 824
 825   CharUnits MaxInlineWidth =
 826       getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
 827
 828   DiagnosticsEngine &Diags = CGM.getDiags();
 829
 830   if (Misaligned) {
 831     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
 832         << (int)TInfo.Width.getQuantity()
 833         << (int)Ptr.getAlignment().getQuantity();
 834   }
 835
 836   if (Oversized) {
 837     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
 838         << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
 839   }
 840
 841   llvm::Value *Order = EmitScalarExpr(E->getOrder());
 842   llvm::Value *Scope =
 843       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
 844
 845   switch (E->getOp()) {
 846   case AtomicExpr::AO__c11_atomic_init:
 847   case AtomicExpr::AO__opencl_atomic_init:
 848     llvm_unreachable("Already handled above with EmitAtomicInit!");
 849
 850   case AtomicExpr::AO__c11_atomic_load:
 851   case AtomicExpr::AO__opencl_atomic_load:
 852   case AtomicExpr::AO__hip_atomic_load:
 853   case AtomicExpr::AO__atomic_load_n:
 854     break;
 855
 856   case AtomicExpr::AO__atomic_load:
 857     Dest = EmitPointerWithAlignment(E->getVal1());
 858     break;
 859
 860   case AtomicExpr::AO__atomic_store:
 861     Val1 = EmitPointerWithAlignment(E->getVal1());
 862     break;
 863
 864   case AtomicExpr::AO__atomic_exchange:
 865     Val1 = EmitPointerWithAlignment(E->getVal1());
 866     Dest = EmitPointerWithAlignment(E->getVal2());
 867     break;
 868
 869   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 870   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 871   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
 872   case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
 873   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
 874   case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
 875   case AtomicExpr::AO__atomic_compare_exchange_n:
 876   case AtomicExpr::AO__atomic_compare_exchange:
 877     Val1 = EmitPointerWithAlignment(E->getVal1());
 878     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 879       Val2 = EmitPointerWithAlignment(E->getVal2());
 880     else
 881       Val2 = EmitValToTemp(*this, E->getVal2());
 882     OrderFail = EmitScalarExpr(E->getOrderFail());
 883     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
 884         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 885       IsWeak = EmitScalarExpr(E->getWeak());
 886     break;
 887
 888   case AtomicExpr::AO__c11_atomic_fetch_add:
 889   case AtomicExpr::AO__c11_atomic_fetch_sub:
 890   case AtomicExpr::AO__hip_atomic_fetch_add:
 891   case AtomicExpr::AO__opencl_atomic_fetch_add:
 892   case AtomicExpr::AO__opencl_atomic_fetch_sub:
 893     if (MemTy->isPointerType()) {
 894       // For pointer arithmetic, we're required to do a bit of math:
 895       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
 896       // ... but only for the C11 builtins. The GNU builtins expect the
 897       // user to multiply by sizeof(T).
 898       QualType Val1Ty = E->getVal1()->getType();
 899       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
 900       CharUnits PointeeIncAmt =
 901           getContext().getTypeSizeInChars(MemTy->getPointeeType());
 902       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
 903       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
 904       Val1 = Temp;
 905       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
 906       break;
 907     }
 908     LLVM_FALLTHROUGH;
 909   case AtomicExpr::AO__atomic_fetch_add:
 910   case AtomicExpr::AO__atomic_fetch_sub:
 911   case AtomicExpr::AO__atomic_add_fetch:
 912   case AtomicExpr::AO__atomic_sub_fetch:
 913     ShouldCastToIntPtrTy = !MemTy->isFloatingType();
 914     LLVM_FALLTHROUGH;
 915
 916   case AtomicExpr::AO__c11_atomic_store:
 917   case AtomicExpr::AO__c11_atomic_exchange:
 918   case AtomicExpr::AO__opencl_atomic_store:
 919   case AtomicExpr::AO__hip_atomic_store:
 920   case AtomicExpr::AO__opencl_atomic_exchange:
 921   case AtomicExpr::AO__hip_atomic_exchange:
 922   case AtomicExpr::AO__atomic_store_n:
 923   case AtomicExpr::AO__atomic_exchange_n:
 924   case AtomicExpr::AO__c11_atomic_fetch_and:
 925   case AtomicExpr::AO__c11_atomic_fetch_or:
 926   case AtomicExpr::AO__c11_atomic_fetch_xor:
 927   case AtomicExpr::AO__c11_atomic_fetch_nand:
 928   case AtomicExpr::AO__c11_atomic_fetch_max:
 929   case AtomicExpr::AO__c11_atomic_fetch_min:
 930   case AtomicExpr::AO__opencl_atomic_fetch_and:
 931   case AtomicExpr::AO__opencl_atomic_fetch_or:
 932   case AtomicExpr::AO__opencl_atomic_fetch_xor:
 933   case AtomicExpr::AO__opencl_atomic_fetch_min:
 934   case AtomicExpr::AO__opencl_atomic_fetch_max:
 935   case AtomicExpr::AO__atomic_fetch_and:
 936   case AtomicExpr::AO__hip_atomic_fetch_and:
 937   case AtomicExpr::AO__atomic_fetch_or:
 938   case AtomicExpr::AO__hip_atomic_fetch_or:
 939   case AtomicExpr::AO__atomic_fetch_xor:
 940   case AtomicExpr::AO__hip_atomic_fetch_xor:
 941   case AtomicExpr::AO__atomic_fetch_nand:
 942   case AtomicExpr::AO__atomic_and_fetch:
 943   case AtomicExpr::AO__atomic_or_fetch:
 944   case AtomicExpr::AO__atomic_xor_fetch:
 945   case AtomicExpr::AO__atomic_nand_fetch:
 946   case AtomicExpr::AO__atomic_max_fetch:
 947   case AtomicExpr::AO__atomic_min_fetch:
 948   case AtomicExpr::AO__atomic_fetch_max:
 949   case AtomicExpr::AO__hip_atomic_fetch_max:
 950   case AtomicExpr::AO__atomic_fetch_min:
 951   case AtomicExpr::AO__hip_atomic_fetch_min:
 952     Val1 = EmitValToTemp(*this, E->getVal1());
 953     break;
 954   }
 955
 956   QualType RValTy = E->getType().getUnqualifiedType();
 957
 958   // The inlined atomics only function on iN types, where N is a power of 2. We
 959   // need to make sure (via temporaries if necessary) that all incoming values
 960   // are compatible.
 961   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
 962   AtomicInfo Atomics(*this, AtomicVal);
 963
 964   if (ShouldCastToIntPtrTy) {
 965     Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
 966     if (Val1.isValid())
 967       Val1 = Atomics.convertToAtomicIntPointer(Val1);
 968     if (Val2.isValid())
 969       Val2 = Atomics.convertToAtomicIntPointer(Val2);
 970   }
 971   if (Dest.isValid()) {
 972     if (ShouldCastToIntPtrTy)
 973       Dest = Atomics.emitCastToAtomicIntPointer(Dest);
 974   } else if (E->isCmpXChg())
 975     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
 976   else if (!RValTy->isVoidType()) {
 977     Dest = Atomics.CreateTempAlloca();
 978     if (ShouldCastToIntPtrTy)
 979       Dest = Atomics.emitCastToAtomicIntPointer(Dest);
 980   }
 981
 982   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
 983   if (UseLibcall) {
 984     bool UseOptimizedLibcall = false;
 985     switch (E->getOp()) {
 986     case AtomicExpr::AO__c11_atomic_init:
 987     case AtomicExpr::AO__opencl_atomic_init:
 988       llvm_unreachable("Already handled above with EmitAtomicInit!");
 989
 990     case AtomicExpr::AO__c11_atomic_fetch_add:
 991     case AtomicExpr::AO__opencl_atomic_fetch_add:
 992     case AtomicExpr::AO__atomic_fetch_add:
 993     case AtomicExpr::AO__hip_atomic_fetch_add:
 994     case AtomicExpr::AO__c11_atomic_fetch_and:
 995     case AtomicExpr::AO__opencl_atomic_fetch_and:
 996     case AtomicExpr::AO__hip_atomic_fetch_and:
 997     case AtomicExpr::AO__atomic_fetch_and:
 998     case AtomicExpr::AO__c11_atomic_fetch_or:
 999     case AtomicExpr::AO__opencl_atomic_fetch_or:
1000     case AtomicExpr::AO__hip_atomic_fetch_or:
1001     case AtomicExpr::AO__atomic_fetch_or:
1002     case AtomicExpr::AO__c11_atomic_fetch_nand:
1003     case AtomicExpr::AO__atomic_fetch_nand:
1004     case AtomicExpr::AO__c11_atomic_fetch_sub:
1005     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1006     case AtomicExpr::AO__atomic_fetch_sub:
1007     case AtomicExpr::AO__c11_atomic_fetch_xor:
1008     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1009     case AtomicExpr::AO__opencl_atomic_fetch_min:
1010     case AtomicExpr::AO__opencl_atomic_fetch_max:
1011     case AtomicExpr::AO__atomic_fetch_xor:
1012     case AtomicExpr::AO__hip_atomic_fetch_xor:
1013     case AtomicExpr::AO__c11_atomic_fetch_max:
1014     case AtomicExpr::AO__c11_atomic_fetch_min:
1015     case AtomicExpr::AO__atomic_add_fetch:
1016     case AtomicExpr::AO__atomic_and_fetch:
1017     case AtomicExpr::AO__atomic_nand_fetch:
1018     case AtomicExpr::AO__atomic_or_fetch:
1019     case AtomicExpr::AO__atomic_sub_fetch:
1020     case AtomicExpr::AO__atomic_xor_fetch:
1021     case AtomicExpr::AO__atomic_fetch_max:
1022     case AtomicExpr::AO__hip_atomic_fetch_max:
1023     case AtomicExpr::AO__atomic_fetch_min:
1024     case AtomicExpr::AO__hip_atomic_fetch_min:
1025     case AtomicExpr::AO__atomic_max_fetch:
1026     case AtomicExpr::AO__atomic_min_fetch:
1027       // For these, only library calls for certain sizes exist.
1028       UseOptimizedLibcall = true;
1029       break;
1030
1031     case AtomicExpr::AO__atomic_load:
1032     case AtomicExpr::AO__atomic_store:
1033     case AtomicExpr::AO__atomic_exchange:
1034     case AtomicExpr::AO__atomic_compare_exchange:
1035       // Use the generic version if we don't know that the operand will be
1036       // suitably aligned for the optimized version.
1037       if (Misaligned)
1038         break;
1039       LLVM_FALLTHROUGH;
1040     case AtomicExpr::AO__c11_atomic_load:
1041     case AtomicExpr::AO__c11_atomic_store:
1042     case AtomicExpr::AO__c11_atomic_exchange:
1043     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1044     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1045     case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1046     case AtomicExpr::AO__opencl_atomic_load:
1047     case AtomicExpr::AO__hip_atomic_load:
1048     case AtomicExpr::AO__opencl_atomic_store:
1049     case AtomicExpr::AO__hip_atomic_store:
1050     case AtomicExpr::AO__opencl_atomic_exchange:
1051     case AtomicExpr::AO__hip_atomic_exchange:
1052     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1053     case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1054     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1055     case AtomicExpr::AO__atomic_load_n:
1056     case AtomicExpr::AO__atomic_store_n:
1057     case AtomicExpr::AO__atomic_exchange_n:
1058     case AtomicExpr::AO__atomic_compare_exchange_n:
1059       // Only use optimized library calls for sizes for which they exist.
1060       // FIXME: Size == 16 optimized library functions exist too.
1061       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1062         UseOptimizedLibcall = true;
1063       break;
1064     }
1065
1066     CallArgList Args;
1067     if (!UseOptimizedLibcall) {
1068       // For non-optimized library calls, the size is the first parameter
1069       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1070                getContext().getSizeType());
1071     }
1072     // Atomic address is the first or second parameter
1073     // The OpenCL atomic library functions only accept pointer arguments to
1074     // generic address space.
1075     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1076       if (!E->isOpenCL())
1077         return V;
1078       auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1079       if (AS == LangAS::opencl_generic)
1080         return V;
1081       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1082       auto T = V->getType();
1083       auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
1084
1085       return getTargetHooks().performAddrSpaceCast(
1086           *this, V, AS, LangAS::opencl_generic, DestType, false);
1087     };
1088
1089     Args.add(RValue::get(CastToGenericAddrSpace(
1090                  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1091              getContext().VoidPtrTy);
1092
1093     std::string LibCallName;
1094     QualType LoweredMemTy =
1095       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1096     QualType RetTy;
1097     bool HaveRetTy = false;
1098     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1099     bool PostOpMinMax = false;
1100     switch (E->getOp()) {
1101     case AtomicExpr::AO__c11_atomic_init:
1102     case AtomicExpr::AO__opencl_atomic_init:
1103       llvm_unreachable("Already handled!");
1104
1105     // There is only one libcall for compare an exchange, because there is no
1106     // optimisation benefit possible from a libcall version of a weak compare
1107     // and exchange.
1108     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1109     //                                void *desired, int success, int failure)
1110     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1111     //                                  int success, int failure)
1112     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1113     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1114     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1115     case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1116     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1117     case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1118     case AtomicExpr::AO__atomic_compare_exchange:
1119     case AtomicExpr::AO__atomic_compare_exchange_n:
1120       LibCallName = "__atomic_compare_exchange";
1121       RetTy = getContext().BoolTy;
1122       HaveRetTy = true;
1123       Args.add(
1124           RValue::get(CastToGenericAddrSpace(
1125               EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1126           getContext().VoidPtrTy);
1127       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1128                         MemTy, E->getExprLoc(), TInfo.Width);
1129       Args.add(RValue::get(Order), getContext().IntTy);
1130       Order = OrderFail;
1131       break;
1132     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1133     //                        int order)
1134     // T __atomic_exchange_N(T *mem, T val, int order)
1135     case AtomicExpr::AO__c11_atomic_exchange:
1136     case AtomicExpr::AO__opencl_atomic_exchange:
1137     case AtomicExpr::AO__atomic_exchange_n:
1138     case AtomicExpr::AO__atomic_exchange:
1139     case AtomicExpr::AO__hip_atomic_exchange:
1140       LibCallName = "__atomic_exchange";
1141       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1142                         MemTy, E->getExprLoc(), TInfo.Width);
1143       break;
1144     // void __atomic_store(size_t size, void *mem, void *val, int order)
1145     // void __atomic_store_N(T *mem, T val, int order)
1146     case AtomicExpr::AO__c11_atomic_store:
1147     case AtomicExpr::AO__opencl_atomic_store:
1148     case AtomicExpr::AO__hip_atomic_store:
1149     case AtomicExpr::AO__atomic_store:
1150     case AtomicExpr::AO__atomic_store_n:
1151       LibCallName = "__atomic_store";
1152       RetTy = getContext().VoidTy;
1153       HaveRetTy = true;
1154       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1155                         MemTy, E->getExprLoc(), TInfo.Width);
1156       break;
1157     // void __atomic_load(size_t size, void *mem, void *return, int order)
1158     // T __atomic_load_N(T *mem, int order)
1159     case AtomicExpr::AO__c11_atomic_load:
1160     case AtomicExpr::AO__opencl_atomic_load:
1161     case AtomicExpr::AO__hip_atomic_load:
1162     case AtomicExpr::AO__atomic_load:
1163     case AtomicExpr::AO__atomic_load_n:
1164       LibCallName = "__atomic_load";
1165       break;
1166     // T __atomic_add_fetch_N(T *mem, T val, int order)
1167     // T __atomic_fetch_add_N(T *mem, T val, int order)
1168     case AtomicExpr::AO__atomic_add_fetch:
1169       PostOp = llvm::Instruction::Add;
1170       LLVM_FALLTHROUGH;
1171     case AtomicExpr::AO__c11_atomic_fetch_add:
1172     case AtomicExpr::AO__opencl_atomic_fetch_add:
1173     case AtomicExpr::AO__atomic_fetch_add:
1174     case AtomicExpr::AO__hip_atomic_fetch_add:
1175       LibCallName = "__atomic_fetch_add";
1176       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1177                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1178       break;
1179     // T __atomic_and_fetch_N(T *mem, T val, int order)
1180     // T __atomic_fetch_and_N(T *mem, T val, int order)
1181     case AtomicExpr::AO__atomic_and_fetch:
1182       PostOp = llvm::Instruction::And;
1183       LLVM_FALLTHROUGH;
1184     case AtomicExpr::AO__c11_atomic_fetch_and:
1185     case AtomicExpr::AO__opencl_atomic_fetch_and:
1186     case AtomicExpr::AO__hip_atomic_fetch_and:
1187     case AtomicExpr::AO__atomic_fetch_and:
1188       LibCallName = "__atomic_fetch_and";
1189       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1190                         MemTy, E->getExprLoc(), TInfo.Width);
1191       break;
1192     // T __atomic_or_fetch_N(T *mem, T val, int order)
1193     // T __atomic_fetch_or_N(T *mem, T val, int order)
1194     case AtomicExpr::AO__atomic_or_fetch:
1195       PostOp = llvm::Instruction::Or;
1196       LLVM_FALLTHROUGH;
1197     case AtomicExpr::AO__c11_atomic_fetch_or:
1198     case AtomicExpr::AO__opencl_atomic_fetch_or:
1199     case AtomicExpr::AO__hip_atomic_fetch_or:
1200     case AtomicExpr::AO__atomic_fetch_or:
1201       LibCallName = "__atomic_fetch_or";
1202       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1203                         MemTy, E->getExprLoc(), TInfo.Width);
1204       break;
1205     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1206     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1207     case AtomicExpr::AO__atomic_sub_fetch:
1208       PostOp = llvm::Instruction::Sub;
1209       LLVM_FALLTHROUGH;
1210     case AtomicExpr::AO__c11_atomic_fetch_sub:
1211     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1212     case AtomicExpr::AO__atomic_fetch_sub:
1213       LibCallName = "__atomic_fetch_sub";
1214       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1215                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1216       break;
1217     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1218     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1219     case AtomicExpr::AO__atomic_xor_fetch:
1220       PostOp = llvm::Instruction::Xor;
1221       LLVM_FALLTHROUGH;
1222     case AtomicExpr::AO__c11_atomic_fetch_xor:
1223     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1224     case AtomicExpr::AO__hip_atomic_fetch_xor:
1225     case AtomicExpr::AO__atomic_fetch_xor:
1226       LibCallName = "__atomic_fetch_xor";
1227       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1228                         MemTy, E->getExprLoc(), TInfo.Width);
1229       break;
1230     case AtomicExpr::AO__atomic_min_fetch:
1231       PostOpMinMax = true;
1232       LLVM_FALLTHROUGH;
1233     case AtomicExpr::AO__c11_atomic_fetch_min:
1234     case AtomicExpr::AO__atomic_fetch_min:
1235     case AtomicExpr::AO__hip_atomic_fetch_min:
1236     case AtomicExpr::AO__opencl_atomic_fetch_min:
1237       LibCallName = E->getValueType()->isSignedIntegerType()
1238                         ? "__atomic_fetch_min"
1239                         : "__atomic_fetch_umin";
1240       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1241                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1242       break;
1243     case AtomicExpr::AO__atomic_max_fetch:
1244       PostOpMinMax = true;
1245       LLVM_FALLTHROUGH;
1246     case AtomicExpr::AO__c11_atomic_fetch_max:
1247     case AtomicExpr::AO__atomic_fetch_max:
1248     case AtomicExpr::AO__hip_atomic_fetch_max:
1249     case AtomicExpr::AO__opencl_atomic_fetch_max:
1250       LibCallName = E->getValueType()->isSignedIntegerType()
1251                         ? "__atomic_fetch_max"
1252                         : "__atomic_fetch_umax";
1253       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1254                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1255       break;
1256     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1257     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1258     case AtomicExpr::AO__atomic_nand_fetch:
1259       PostOp = llvm::Instruction::And; // the NOT is special cased below
1260       LLVM_FALLTHROUGH;
1261     case AtomicExpr::AO__c11_atomic_fetch_nand:
1262     case AtomicExpr::AO__atomic_fetch_nand:
1263       LibCallName = "__atomic_fetch_nand";
1264       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1265                         MemTy, E->getExprLoc(), TInfo.Width);
1266       break;
1267     }
1268
1269     if (E->isOpenCL()) {
1270       LibCallName = std::string("__opencl") +
1271           StringRef(LibCallName).drop_front(1).str();
1272
1273     }
1274     // Optimized functions have the size in their name.
1275     if (UseOptimizedLibcall)
1276       LibCallName += "_" + llvm::utostr(Size);
1277     // By default, assume we return a value of the atomic type.
1278     if (!HaveRetTy) {
1279       if (UseOptimizedLibcall) {
1280         // Value is returned directly.
1281         // The function returns an appropriately sized integer type.
1282         RetTy = getContext().getIntTypeForBitwidth(
1283             getContext().toBits(TInfo.Width), /*Signed=*/false);
1284       } else {
1285         // Value is returned through parameter before the order.
1286         RetTy = getContext().VoidTy;
1287         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1288                  getContext().VoidPtrTy);
1289       }
1290     }
1291     // order is always the last parameter
1292     Args.add(RValue::get(Order),
1293              getContext().IntTy);
1294     if (E->isOpenCL())
1295       Args.add(RValue::get(Scope), getContext().IntTy);
1296
1297     // PostOp is only needed for the atomic_*_fetch operations, and
1298     // thus is only needed for and implemented in the
1299     // UseOptimizedLibcall codepath.
1300     assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1301
1302     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1303     // The value is returned directly from the libcall.
1304     if (E->isCmpXChg())
1305       return Res;
1306
1307     // The value is returned directly for optimized libcalls but the expr
1308     // provided an out-param.
1309     if (UseOptimizedLibcall && Res.getScalarVal()) {
1310       llvm::Value *ResVal = Res.getScalarVal();
1311       if (PostOpMinMax) {
1312         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1313         ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1314                                       E->getValueType()->isSignedIntegerType(),
1315                                       ResVal, LoadVal1);
1316       } else if (PostOp) {
1317         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1318         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1319       }
1320       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1321         ResVal = Builder.CreateNot(ResVal);
1322
1323       Builder.CreateStore(
1324           ResVal,
1325           Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1326     }
1327
1328     if (RValTy->isVoidType())
1329       return RValue::get(nullptr);
1330
1331     return convertTempToRValue(
1332         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1333         RValTy, E->getExprLoc());
1334   }
1335
1336   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1337                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1338                  E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1339                  E->getOp() == AtomicExpr::AO__atomic_store ||
1340                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1341   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1342                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1343                 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1344                 E->getOp() == AtomicExpr::AO__atomic_load ||
1345                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1346
1347   if (isa<llvm::ConstantInt>(Order)) {
1348     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1349     // We should not ever get to a case where the ordering isn't a valid C ABI
1350     // value, but it's hard to enforce that in general.
1351     if (llvm::isValidAtomicOrderingCABI(ord))
1352       switch ((llvm::AtomicOrderingCABI)ord) {
1353       case llvm::AtomicOrderingCABI::relaxed:
1354         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1355                      llvm::AtomicOrdering::Monotonic, Scope);
1356         break;
1357       case llvm::AtomicOrderingCABI::consume:
1358       case llvm::AtomicOrderingCABI::acquire:
1359         if (IsStore)
1360           break; // Avoid crashing on code with undefined behavior
1361         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1362                      llvm::AtomicOrdering::Acquire, Scope);
1363         break;
1364       case llvm::AtomicOrderingCABI::release:
1365         if (IsLoad)
1366           break; // Avoid crashing on code with undefined behavior
1367         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1368                      llvm::AtomicOrdering::Release, Scope);
1369         break;
1370       case llvm::AtomicOrderingCABI::acq_rel:
1371         if (IsLoad || IsStore)
1372           break; // Avoid crashing on code with undefined behavior
1373         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1374                      llvm::AtomicOrdering::AcquireRelease, Scope);
1375         break;
1376       case llvm::AtomicOrderingCABI::seq_cst:
1377         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1378                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1379         break;
1380       }
1381     if (RValTy->isVoidType())
1382       return RValue::get(nullptr);
1383
1384     return convertTempToRValue(
1385         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1386                                         Dest.getAddressSpace())),
1387         RValTy, E->getExprLoc());
1388   }
1389
1390   // Long case, when Order isn't obviously constant.
1391
1392   // Create all the relevant BB's
1393   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1394                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1395                    *SeqCstBB = nullptr;
1396   MonotonicBB = createBasicBlock("monotonic", CurFn);
1397   if (!IsStore)
1398     AcquireBB = createBasicBlock("acquire", CurFn);
1399   if (!IsLoad)
1400     ReleaseBB = createBasicBlock("release", CurFn);
1401   if (!IsLoad && !IsStore)
1402     AcqRelBB = createBasicBlock("acqrel", CurFn);
1403   SeqCstBB = createBasicBlock("seqcst", CurFn);
1404   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1405
1406   // Create the switch for the split
1407   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1408   // doesn't matter unless someone is crazy enough to use something that
1409   // doesn't fold to a constant for the ordering.
1410   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1411   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1412
1413   // Emit all the different atomics
1414   Builder.SetInsertPoint(MonotonicBB);
1415   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1416                llvm::AtomicOrdering::Monotonic, Scope);
1417   Builder.CreateBr(ContBB);
1418   if (!IsStore) {
1419     Builder.SetInsertPoint(AcquireBB);
1420     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1421                  llvm::AtomicOrdering::Acquire, Scope);
1422     Builder.CreateBr(ContBB);
1423     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1424                 AcquireBB);
1425     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1426                 AcquireBB);
1427   }
1428   if (!IsLoad) {
1429     Builder.SetInsertPoint(ReleaseBB);
1430     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1431                  llvm::AtomicOrdering::Release, Scope);
1432     Builder.CreateBr(ContBB);
1433     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1434                 ReleaseBB);
1435   }
1436   if (!IsLoad && !IsStore) {
1437     Builder.SetInsertPoint(AcqRelBB);
1438     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1439                  llvm::AtomicOrdering::AcquireRelease, Scope);
1440     Builder.CreateBr(ContBB);
1441     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1442                 AcqRelBB);
1443   }
1444   Builder.SetInsertPoint(SeqCstBB);
1445   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1446                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1447   Builder.CreateBr(ContBB);
1448   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1449               SeqCstBB);
1450
1451   // Cleanup and return
1452   Builder.SetInsertPoint(ContBB);
1453   if (RValTy->isVoidType())
1454     return RValue::get(nullptr);
1455
1456   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1457   return convertTempToRValue(
1458       Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1459                                       Dest.getAddressSpace())),
1460       RValTy, E->getExprLoc());
1461 }
1462
1463 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1464   unsigned addrspace =
1465     cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1466   llvm::IntegerType *ty =
1467     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1468   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1469 }
1470
1471 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1472   llvm::Type *Ty = Addr.getElementType();
1473   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1474   if (SourceSizeInBits != AtomicSizeInBits) {
1475     Address Tmp = CreateTempAlloca();
1476     CGF.Builder.CreateMemCpy(Tmp, Addr,
1477                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1478     Addr = Tmp;
1479   }
1480
1481   return emitCastToAtomicIntPointer(Addr);
1482 }
1483
1484 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1485                                              AggValueSlot resultSlot,
1486                                              SourceLocation loc,
1487                                              bool asValue) const {
1488   if (LVal.isSimple()) {
1489     if (EvaluationKind == TEK_Aggregate)
1490       return resultSlot.asRValue();
1491
1492     // Drill into the padding structure if we have one.
1493     if (hasPadding())
1494       addr = CGF.Builder.CreateStructGEP(addr, 0);
1495
1496     // Otherwise, just convert the temporary to an r-value using the
1497     // normal conversion routine.
1498     return CGF.convertTempToRValue(addr, getValueType(), loc);
1499   }
1500   if (!asValue)
1501     // Get RValue from temp memory as atomic for non-simple lvalues
1502     return RValue::get(CGF.Builder.CreateLoad(addr));
1503   if (LVal.isBitField())
1504     return CGF.EmitLoadOfBitfieldLValue(
1505         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1506                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1507   if (LVal.isVectorElt())
1508     return CGF.EmitLoadOfLValue(
1509         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1510                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1511   assert(LVal.isExtVectorElt());
1512   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1513       addr, LVal.getExtVectorElts(), LVal.getType(),
1514       LVal.getBaseInfo(), TBAAAccessInfo()));
1515 }
1516
1517 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1518                                              AggValueSlot ResultSlot,
1519                                              SourceLocation Loc,
1520                                              bool AsValue) const {
1521   // Try not to in some easy cases.
1522   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1523   if (getEvaluationKind() == TEK_Scalar &&
1524       (((!LVal.isBitField() ||
1525          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1526         !hasPadding()) ||
1527        !AsValue)) {
1528     auto *ValTy = AsValue
1529                       ? CGF.ConvertTypeForMem(ValueTy)
1530                       : getAtomicAddress().getType()->getPointerElementType();
1531     if (ValTy->isIntegerTy()) {
1532       assert(IntVal->getType() == ValTy && "Different integer types.");
1533       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1534     } else if (ValTy->isPointerTy())
1535       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1536     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1537       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1538   }
1539
1540   // Create a temporary.  This needs to be big enough to hold the
1541   // atomic integer.
1542   Address Temp = Address::invalid();
1543   bool TempIsVolatile = false;
1544   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1545     assert(!ResultSlot.isIgnored());
1546     Temp = ResultSlot.getAddress();
1547     TempIsVolatile = ResultSlot.isVolatile();
1548   } else {
1549     Temp = CreateTempAlloca();
1550   }
1551
1552   // Slam the integer into the temporary.
1553   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1554   CGF.Builder.CreateStore(IntVal, CastTemp)
1555       ->setVolatile(TempIsVolatile);
1556
1557   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1558 }
1559
1560 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1561                                        llvm::AtomicOrdering AO, bool) {
1562   // void __atomic_load(size_t size, void *mem, void *return, int order);
1563   CallArgList Args;
1564   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1565   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1566            CGF.getContext().VoidPtrTy);
1567   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1568            CGF.getContext().VoidPtrTy);
1569   Args.add(
1570       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1571       CGF.getContext().IntTy);
1572   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1573 }
1574
1575 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1576                                           bool IsVolatile) {
1577   // Okay, we're doing this natively.
1578   Address Addr = getAtomicAddressAsAtomicIntPointer();
1579   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1580   Load->setAtomic(AO);
1581
1582   // Other decoration.
1583   if (IsVolatile)
1584     Load->setVolatile(true);
1585   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1586   return Load;
1587 }
1588
1589 /// An LValue is a candidate for having its loads and stores be made atomic if
1590 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1591 /// performing such an operation can be performed without a libcall.
1592 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1593   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1594   AtomicInfo AI(*this, LV);
1595   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1596   // An atomic is inline if we don't need to use a libcall.
1597   bool AtomicIsInline = !AI.shouldUseLibcall();
1598   // MSVC doesn't seem to do this for types wider than a pointer.
1599   if (getContext().getTypeSize(LV.getType()) >
1600       getContext().getTypeSize(getContext().getIntPtrType()))
1601     return false;
1602   return IsVolatile && AtomicIsInline;
1603 }
1604
1605 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1606                                        AggValueSlot Slot) {
1607   llvm::AtomicOrdering AO;
1608   bool IsVolatile = LV.isVolatileQualified();
1609   if (LV.getType()->isAtomicType()) {
1610     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1611   } else {
1612     AO = llvm::AtomicOrdering::Acquire;
1613     IsVolatile = true;
1614   }
1615   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1616 }
1617
1618 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1619                                   bool AsValue, llvm::AtomicOrdering AO,
1620                                   bool IsVolatile) {
1621   // Check whether we should use a library call.
1622   if (shouldUseLibcall()) {
1623     Address TempAddr = Address::invalid();
1624     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1625       assert(getEvaluationKind() == TEK_Aggregate);
1626       TempAddr = ResultSlot.getAddress();
1627     } else
1628       TempAddr = CreateTempAlloca();
1629
1630     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1631
1632     // Okay, turn that back into the original value or whole atomic (for
1633     // non-simple lvalues) type.
1634     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1635   }
1636
1637   // Okay, we're doing this natively.
1638   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1639
1640   // If we're ignoring an aggregate return, don't do anything.
1641   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1642     return RValue::getAggregate(Address::invalid(), false);
1643
1644   // Okay, turn that back into the original value or atomic (for non-simple
1645   // lvalues) type.
1646   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1647 }
1648
1649 /// Emit a load from an l-value of atomic type.  Note that the r-value
1650 /// we produce is an r-value of the atomic *value* type.
1651 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1652                                        llvm::AtomicOrdering AO, bool IsVolatile,
1653                                        AggValueSlot resultSlot) {
1654   AtomicInfo Atomics(*this, src);
1655   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1656                                 IsVolatile);
1657 }
1658
1659 /// Copy an r-value into memory as part of storing to an atomic type.
1660 /// This needs to create a bit-pattern suitable for atomic operations.
1661 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1662   assert(LVal.isSimple());
1663   // If we have an r-value, the rvalue should be of the atomic type,
1664   // which means that the caller is responsible for having zeroed
1665   // any padding.  Just do an aggregate copy of that type.
1666   if (rvalue.isAggregate()) {
1667     LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1668     LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1669                                     getAtomicType());
1670     bool IsVolatile = rvalue.isVolatileQualified() ||
1671                       LVal.isVolatileQualified();
1672     CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1673                           AggValueSlot::DoesNotOverlap, IsVolatile);
1674     return;
1675   }
1676
1677   // Okay, otherwise we're copying stuff.
1678
1679   // Zero out the buffer if necessary.
1680   emitMemSetZeroIfNecessary();
1681
1682   // Drill past the padding if present.
1683   LValue TempLVal = projectValue();
1684
1685   // Okay, store the rvalue in.
1686   if (rvalue.isScalar()) {
1687     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1688   } else {
1689     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1690   }
1691 }
1692
1693
1694 /// Materialize an r-value into memory for the purposes of storing it
1695 /// to an atomic type.
1696 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1697   // Aggregate r-values are already in memory, and EmitAtomicStore
1698   // requires them to be values of the atomic type.
1699   if (rvalue.isAggregate())
1700     return rvalue.getAggregateAddress();
1701
1702   // Otherwise, make a temporary and materialize into it.
1703   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1704   AtomicInfo Atomics(CGF, TempLV);
1705   Atomics.emitCopyIntoMemory(rvalue);
1706   return TempLV.getAddress(CGF);
1707 }
1708
1709 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1710   // If we've got a scalar value of the right size, try to avoid going
1711   // through memory.
1712   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1713     llvm::Value *Value = RVal.getScalarVal();
1714     if (isa<llvm::IntegerType>(Value->getType()))
1715       return CGF.EmitToMemory(Value, ValueTy);
1716     else {
1717       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1718           CGF.getLLVMContext(),
1719           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1720       if (isa<llvm::PointerType>(Value->getType()))
1721         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1722       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1723         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1724     }
1725   }
1726   // Otherwise, we need to go through memory.
1727   // Put the r-value in memory.
1728   Address Addr = materializeRValue(RVal);
1729
1730   // Cast the temporary to the atomic int type and pull a value out.
1731   Addr = emitCastToAtomicIntPointer(Addr);
1732   return CGF.Builder.CreateLoad(Addr);
1733 }
1734
1735 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1736     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1737     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1738   // Do the atomic store.
1739   Address Addr = getAtomicAddressAsAtomicIntPointer();
1740   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1741                                                ExpectedVal, DesiredVal,
1742                                                Success, Failure);
1743   // Other decoration.
1744   Inst->setVolatile(LVal.isVolatileQualified());
1745   Inst->setWeak(IsWeak);
1746
1747   // Okay, turn that back into the original value type.
1748   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1749   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1750   return std::make_pair(PreviousVal, SuccessFailureVal);
1751 }
1752
1753 llvm::Value *
1754 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1755                                              llvm::Value *DesiredAddr,
1756                                              llvm::AtomicOrdering Success,
1757                                              llvm::AtomicOrdering Failure) {
1758   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1759   // void *desired, int success, int failure);
1760   CallArgList Args;
1761   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1762   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1763            CGF.getContext().VoidPtrTy);
1764   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1765            CGF.getContext().VoidPtrTy);
1766   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1767            CGF.getContext().VoidPtrTy);
1768   Args.add(RValue::get(
1769                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1770            CGF.getContext().IntTy);
1771   Args.add(RValue::get(
1772                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1773            CGF.getContext().IntTy);
1774   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1775                                               CGF.getContext().BoolTy, Args);
1776
1777   return SuccessFailureRVal.getScalarVal();
1778 }
1779
1780 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1781     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1782     llvm::AtomicOrdering Failure, bool IsWeak) {
1783   // Check whether we should use a library call.
1784   if (shouldUseLibcall()) {
1785     // Produce a source address.
1786     Address ExpectedAddr = materializeRValue(Expected);
1787     Address DesiredAddr = materializeRValue(Desired);
1788     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1789                                                  DesiredAddr.getPointer(),
1790                                                  Success, Failure);
1791     return std::make_pair(
1792         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1793                                   SourceLocation(), /*AsValue=*/false),
1794         Res);
1795   }
1796
1797   // If we've got a scalar value of the right size, try to avoid going
1798   // through memory.
1799   auto *ExpectedVal = convertRValueToInt(Expected);
1800   auto *DesiredVal = convertRValueToInt(Desired);
1801   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1802                                          Failure, IsWeak);
1803   return std::make_pair(
1804       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1805                                 SourceLocation(), /*AsValue=*/false),
1806       Res.second);
1807 }
1808
1809 static void
1810 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1811                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1812                       Address DesiredAddr) {
1813   RValue UpRVal;
1814   LValue AtomicLVal = Atomics.getAtomicLValue();
1815   LValue DesiredLVal;
1816   if (AtomicLVal.isSimple()) {
1817     UpRVal = OldRVal;
1818     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1819   } else {
1820     // Build new lvalue for temp address.
1821     Address Ptr = Atomics.materializeRValue(OldRVal);
1822     LValue UpdateLVal;
1823     if (AtomicLVal.isBitField()) {
1824       UpdateLVal =
1825           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1826                                AtomicLVal.getType(),
1827                                AtomicLVal.getBaseInfo(),
1828                                AtomicLVal.getTBAAInfo());
1829       DesiredLVal =
1830           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1831                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1832                                AtomicLVal.getTBAAInfo());
1833     } else if (AtomicLVal.isVectorElt()) {
1834       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1835                                          AtomicLVal.getType(),
1836                                          AtomicLVal.getBaseInfo(),
1837                                          AtomicLVal.getTBAAInfo());
1838       DesiredLVal = LValue::MakeVectorElt(
1839           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1840           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1841     } else {
1842       assert(AtomicLVal.isExtVectorElt());
1843       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1844                                             AtomicLVal.getType(),
1845                                             AtomicLVal.getBaseInfo(),
1846                                             AtomicLVal.getTBAAInfo());
1847       DesiredLVal = LValue::MakeExtVectorElt(
1848           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1849           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1850     }
1851     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1852   }
1853   // Store new value in the corresponding memory area.
1854   RValue NewRVal = UpdateOp(UpRVal);
1855   if (NewRVal.isScalar()) {
1856     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1857   } else {
1858     assert(NewRVal.isComplex());
1859     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1860                            /*isInit=*/false);
1861   }
1862 }
1863
1864 void AtomicInfo::EmitAtomicUpdateLibcall(
1865     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1866     bool IsVolatile) {
1867   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1868
1869   Address ExpectedAddr = CreateTempAlloca();
1870
1871   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1872   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1873   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1874   CGF.EmitBlock(ContBB);
1875   Address DesiredAddr = CreateTempAlloca();
1876   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1877       requiresMemSetZero(getAtomicAddress().getElementType())) {
1878     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1879     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1880   }
1881   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1882                                            AggValueSlot::ignored(),
1883                                            SourceLocation(), /*AsValue=*/false);
1884   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1885   auto *Res =
1886       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1887                                        DesiredAddr.getPointer(),
1888                                        AO, Failure);
1889   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1890   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1891 }
1892
1893 void AtomicInfo::EmitAtomicUpdateOp(
1894     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1895     bool IsVolatile) {
1896   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1897
1898   // Do the atomic load.
1899   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1900   // For non-simple lvalues perform compare-and-swap procedure.
1901   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1902   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1903   auto *CurBB = CGF.Builder.GetInsertBlock();
1904   CGF.EmitBlock(ContBB);
1905   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1906                                              /*NumReservedValues=*/2);
1907   PHI->addIncoming(OldVal, CurBB);
1908   Address NewAtomicAddr = CreateTempAlloca();
1909   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1910   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1911       requiresMemSetZero(getAtomicAddress().getElementType())) {
1912     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1913   }
1914   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1915                                            SourceLocation(), /*AsValue=*/false);
1916   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1917   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1918   // Try to write new value using cmpxchg operation.
1919   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1920   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1921   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1922   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1923 }
1924
1925 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1926                                   RValue UpdateRVal, Address DesiredAddr) {
1927   LValue AtomicLVal = Atomics.getAtomicLValue();
1928   LValue DesiredLVal;
1929   // Build new lvalue for temp address.
1930   if (AtomicLVal.isBitField()) {
1931     DesiredLVal =
1932         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1933                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1934                              AtomicLVal.getTBAAInfo());
1935   } else if (AtomicLVal.isVectorElt()) {
1936     DesiredLVal =
1937         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1938                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1939                               AtomicLVal.getTBAAInfo());
1940   } else {
1941     assert(AtomicLVal.isExtVectorElt());
1942     DesiredLVal = LValue::MakeExtVectorElt(
1943         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1944         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1945   }
1946   // Store new value in the corresponding memory area.
1947   assert(UpdateRVal.isScalar());
1948   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1949 }
1950
1951 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1952                                          RValue UpdateRVal, bool IsVolatile) {
1953   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1954
1955   Address ExpectedAddr = CreateTempAlloca();
1956
1957   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1958   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1959   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1960   CGF.EmitBlock(ContBB);
1961   Address DesiredAddr = CreateTempAlloca();
1962   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1963       requiresMemSetZero(getAtomicAddress().getElementType())) {
1964     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1965     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1966   }
1967   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1968   auto *Res =
1969       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1970                                        DesiredAddr.getPointer(),
1971                                        AO, Failure);
1972   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1973   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1974 }
1975
1976 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1977                                     bool IsVolatile) {
1978   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1979
1980   // Do the atomic load.
1981   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1982   // For non-simple lvalues perform compare-and-swap procedure.
1983   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1984   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1985   auto *CurBB = CGF.Builder.GetInsertBlock();
1986   CGF.EmitBlock(ContBB);
1987   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1988                                              /*NumReservedValues=*/2);
1989   PHI->addIncoming(OldVal, CurBB);
1990   Address NewAtomicAddr = CreateTempAlloca();
1991   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1992   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1993       requiresMemSetZero(getAtomicAddress().getElementType())) {
1994     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1995   }
1996   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1997   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1998   // Try to write new value using cmpxchg operation.
1999   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2000   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2001   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2002   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2003 }
2004
2005 void AtomicInfo::EmitAtomicUpdate(
2006     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2007     bool IsVolatile) {
2008   if (shouldUseLibcall()) {
2009     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2010   } else {
2011     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2012   }
2013 }
2014
2015 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2016                                   bool IsVolatile) {
2017   if (shouldUseLibcall()) {
2018     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2019   } else {
2020     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2021   }
2022 }
2023
2024 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2025                                       bool isInit) {
2026   bool IsVolatile = lvalue.isVolatileQualified();
2027   llvm::AtomicOrdering AO;
2028   if (lvalue.getType()->isAtomicType()) {
2029     AO = llvm::AtomicOrdering::SequentiallyConsistent;
2030   } else {
2031     AO = llvm::AtomicOrdering::Release;
2032     IsVolatile = true;
2033   }
2034   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2035 }
2036
2037 /// Emit a store to an l-value of atomic type.
2038 ///
2039 /// Note that the r-value is expected to be an r-value *of the atomic
2040 /// type*; this means that for aggregate r-values, it should include
2041 /// storage for any padding that was necessary.
2042 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2043                                       llvm::AtomicOrdering AO, bool IsVolatile,
2044                                       bool isInit) {
2045   // If this is an aggregate r-value, it should agree in type except
2046   // maybe for address-space qualification.
2047   assert(!rvalue.isAggregate() ||
2048          rvalue.getAggregateAddress().getElementType() ==
2049              dest.getAddress(*this).getElementType());
2050
2051   AtomicInfo atomics(*this, dest);
2052   LValue LVal = atomics.getAtomicLValue();
2053
2054   // If this is an initialization, just put the value there normally.
2055   if (LVal.isSimple()) {
2056     if (isInit) {
2057       atomics.emitCopyIntoMemory(rvalue);
2058       return;
2059     }
2060
2061     // Check whether we should use a library call.
2062     if (atomics.shouldUseLibcall()) {
2063       // Produce a source address.
2064       Address srcAddr = atomics.materializeRValue(rvalue);
2065
2066       // void __atomic_store(size_t size, void *mem, void *val, int order)
2067       CallArgList args;
2068       args.add(RValue::get(atomics.getAtomicSizeValue()),
2069                getContext().getSizeType());
2070       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2071                getContext().VoidPtrTy);
2072       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2073                getContext().VoidPtrTy);
2074       args.add(
2075           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2076           getContext().IntTy);
2077       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2078       return;
2079     }
2080
2081     // Okay, we're doing this natively.
2082     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2083
2084     // Do the atomic store.
2085     Address addr =
2086         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2087     intValue = Builder.CreateIntCast(
2088         intValue, addr.getElementType(), /*isSigned=*/false);
2089     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2090
2091     if (AO == llvm::AtomicOrdering::Acquire)
2092       AO = llvm::AtomicOrdering::Monotonic;
2093     else if (AO == llvm::AtomicOrdering::AcquireRelease)
2094       AO = llvm::AtomicOrdering::Release;
2095     // Initializations don't need to be atomic.
2096     if (!isInit)
2097       store->setAtomic(AO);
2098
2099     // Other decoration.
2100     if (IsVolatile)
2101       store->setVolatile(true);
2102     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2103     return;
2104   }
2105
2106   // Emit simple atomic update operation.
2107   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2108 }
2109
2110 /// Emit a compare-and-exchange op for atomic type.
2111 ///
2112 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2113     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2114     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2115     AggValueSlot Slot) {
2116   // If this is an aggregate r-value, it should agree in type except
2117   // maybe for address-space qualification.
2118   assert(!Expected.isAggregate() ||
2119          Expected.getAggregateAddress().getElementType() ==
2120              Obj.getAddress(*this).getElementType());
2121   assert(!Desired.isAggregate() ||
2122          Desired.getAggregateAddress().getElementType() ==
2123              Obj.getAddress(*this).getElementType());
2124   AtomicInfo Atomics(*this, Obj);
2125
2126   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2127                                            IsWeak);
2128 }
2129
2130 void CodeGenFunction::EmitAtomicUpdate(
2131     LValue LVal, llvm::AtomicOrdering AO,
2132     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2133   AtomicInfo Atomics(*this, LVal);
2134   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2135 }
2136
2137 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2138   AtomicInfo atomics(*this, dest);
2139
2140   switch (atomics.getEvaluationKind()) {
2141   case TEK_Scalar: {
2142     llvm::Value *value = EmitScalarExpr(init);
2143     atomics.emitCopyIntoMemory(RValue::get(value));
2144     return;
2145   }
2146
2147   case TEK_Complex: {
2148     ComplexPairTy value = EmitComplexExpr(init);
2149     atomics.emitCopyIntoMemory(RValue::getComplex(value));
2150     return;
2151   }
2152
2153   case TEK_Aggregate: {
2154     // Fix up the destination if the initializer isn't an expression
2155     // of atomic type.
2156     bool Zeroed = false;
2157     if (!init->getType()->isAtomicType()) {
2158       Zeroed = atomics.emitMemSetZeroIfNecessary();
2159       dest = atomics.projectValue();
2160     }
2161
2162     // Evaluate the expression directly into the destination.
2163     AggValueSlot slot = AggValueSlot::forLValue(
2164         dest, *this, AggValueSlot::IsNotDestructed,
2165         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2166         AggValueSlot::DoesNotOverlap,
2167         Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2168
2169     EmitAggExpr(init, slot);
2170     return;
2171   }
2172   }
2173   llvm_unreachable("bad evaluation kind");
2174 }