contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp

   1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file contains the code for emitting atomic operations.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "CodeGenFunction.h"
  15 #include "CGCall.h"
  16 #include "CGRecordLayout.h"
  17 #include "CodeGenModule.h"
  18 #include "clang/AST/ASTContext.h"
  19 #include "clang/CodeGen/CGFunctionInfo.h"
  20 #include "llvm/ADT/StringExtras.h"
  21 #include "llvm/IR/DataLayout.h"
  22 #include "llvm/IR/Intrinsics.h"
  23 #include "llvm/IR/Operator.h"
  24
  25 using namespace clang;
  26 using namespace CodeGen;
  27
  28 namespace {
  29   class AtomicInfo {
  30     CodeGenFunction &CGF;
  31     QualType AtomicTy;
  32     QualType ValueTy;
  33     uint64_t AtomicSizeInBits;
  34     uint64_t ValueSizeInBits;
  35     CharUnits AtomicAlign;
  36     CharUnits ValueAlign;
  37     CharUnits LValueAlign;
  38     TypeEvaluationKind EvaluationKind;
  39     bool UseLibcall;
  40     LValue LVal;
  41     CGBitFieldInfo BFI;
  42   public:
  43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
  44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
  45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
  46       assert(!lvalue.isGlobalReg());
  47       ASTContext &C = CGF.getContext();
  48       if (lvalue.isSimple()) {
  49         AtomicTy = lvalue.getType();
  50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
  51           ValueTy = ATy->getValueType();
  52         else
  53           ValueTy = AtomicTy;
  54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
  55
  56         uint64_t ValueAlignInBits;
  57         uint64_t AtomicAlignInBits;
  58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
  59         ValueSizeInBits = ValueTI.Width;
  60         ValueAlignInBits = ValueTI.Align;
  61
  62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
  63         AtomicSizeInBits = AtomicTI.Width;
  64         AtomicAlignInBits = AtomicTI.Align;
  65
  66         assert(ValueSizeInBits <= AtomicSizeInBits);
  67         assert(ValueAlignInBits <= AtomicAlignInBits);
  68
  69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
  70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
  71         if (lvalue.getAlignment().isZero())
  72           lvalue.setAlignment(AtomicAlign);
  73
  74         LVal = lvalue;
  75       } else if (lvalue.isBitField()) {
  76         ValueTy = lvalue.getType();
  77         ValueSizeInBits = C.getTypeSize(ValueTy);
  78         auto &OrigBFI = lvalue.getBitFieldInfo();
  79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
  80         AtomicSizeInBits = C.toBits(
  81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
  82                 .RoundUpToAlignment(lvalue.getAlignment()));
  83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
  84         auto OffsetInChars =
  85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
  86             lvalue.getAlignment();
  87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
  88             VoidPtrAddr, OffsetInChars.getQuantity());
  89         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
  90             VoidPtrAddr,
  91             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
  92             "atomic_bitfield_base");
  93         BFI = OrigBFI;
  94         BFI.Offset = Offset;
  95         BFI.StorageSize = AtomicSizeInBits;
  96         BFI.StorageOffset += OffsetInChars;
  97         LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
  98                                     BFI, lvalue.getType(),
  99                                     lvalue.getAlignmentSource());
 100         LVal.setTBAAInfo(lvalue.getTBAAInfo());
 101         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
 102         if (AtomicTy.isNull()) {
 103           llvm::APInt Size(
 104               /*numBits=*/32,
 105               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
 106           AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
 107                                             /*IndexTypeQuals=*/0);
 108         }
 109         AtomicAlign = ValueAlign = lvalue.getAlignment();
 110       } else if (lvalue.isVectorElt()) {
 111         ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
 112         ValueSizeInBits = C.getTypeSize(ValueTy);
 113         AtomicTy = lvalue.getType();
 114         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 115         AtomicAlign = ValueAlign = lvalue.getAlignment();
 116         LVal = lvalue;
 117       } else {
 118         assert(lvalue.isExtVectorElt());
 119         ValueTy = lvalue.getType();
 120         ValueSizeInBits = C.getTypeSize(ValueTy);
 121         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
 122             lvalue.getType(), lvalue.getExtVectorAddress()
 123                                   .getElementType()->getVectorNumElements());
 124         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 125         AtomicAlign = ValueAlign = lvalue.getAlignment();
 126         LVal = lvalue;
 127       }
 128       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
 129           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
 130     }
 131
 132     QualType getAtomicType() const { return AtomicTy; }
 133     QualType getValueType() const { return ValueTy; }
 134     CharUnits getAtomicAlignment() const { return AtomicAlign; }
 135     CharUnits getValueAlignment() const { return ValueAlign; }
 136     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
 137     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
 138     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
 139     bool shouldUseLibcall() const { return UseLibcall; }
 140     const LValue &getAtomicLValue() const { return LVal; }
 141     llvm::Value *getAtomicPointer() const {
 142       if (LVal.isSimple())
 143         return LVal.getPointer();
 144       else if (LVal.isBitField())
 145         return LVal.getBitFieldPointer();
 146       else if (LVal.isVectorElt())
 147         return LVal.getVectorPointer();
 148       assert(LVal.isExtVectorElt());
 149       return LVal.getExtVectorPointer();
 150     }
 151     Address getAtomicAddress() const {
 152       return Address(getAtomicPointer(), getAtomicAlignment());
 153     }
 154
 155     Address getAtomicAddressAsAtomicIntPointer() const {
 156       return emitCastToAtomicIntPointer(getAtomicAddress());
 157     }
 158
 159     /// Is the atomic size larger than the underlying value type?
 160     ///
 161     /// Note that the absence of padding does not mean that atomic
 162     /// objects are completely interchangeable with non-atomic
 163     /// objects: we might have promoted the alignment of a type
 164     /// without making it bigger.
 165     bool hasPadding() const {
 166       return (ValueSizeInBits != AtomicSizeInBits);
 167     }
 168
 169     bool emitMemSetZeroIfNecessary() const;
 170
 171     llvm::Value *getAtomicSizeValue() const {
 172       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
 173       return CGF.CGM.getSize(size);
 174     }
 175
 176     /// Cast the given pointer to an integer pointer suitable for atomic
 177     /// operations if the source.
 178     Address emitCastToAtomicIntPointer(Address Addr) const;
 179
 180     /// If Addr is compatible with the iN that will be used for an atomic
 181     /// operation, bitcast it. Otherwise, create a temporary that is suitable
 182     /// and copy the value across.
 183     Address convertToAtomicIntPointer(Address Addr) const;
 184
 185     /// Turn an atomic-layout object into an r-value.
 186     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
 187                                      SourceLocation loc, bool AsValue) const;
 188
 189     /// \brief Converts a rvalue to integer value.
 190     llvm::Value *convertRValueToInt(RValue RVal) const;
 191
 192     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
 193                                      AggValueSlot ResultSlot,
 194                                      SourceLocation Loc, bool AsValue) const;
 195
 196     /// Copy an atomic r-value into atomic-layout memory.
 197     void emitCopyIntoMemory(RValue rvalue) const;
 198
 199     /// Project an l-value down to the value field.
 200     LValue projectValue() const {
 201       assert(LVal.isSimple());
 202       Address addr = getAtomicAddress();
 203       if (hasPadding())
 204         addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
 205
 206       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
 207                               LVal.getAlignmentSource(), LVal.getTBAAInfo());
 208     }
 209
 210     /// \brief Emits atomic load.
 211     /// \returns Loaded value.
 212     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
 213                           bool AsValue, llvm::AtomicOrdering AO,
 214                           bool IsVolatile);
 215
 216     /// \brief Emits atomic compare-and-exchange sequence.
 217     /// \param Expected Expected value.
 218     /// \param Desired Desired value.
 219     /// \param Success Atomic ordering for success operation.
 220     /// \param Failure Atomic ordering for failed operation.
 221     /// \param IsWeak true if atomic operation is weak, false otherwise.
 222     /// \returns Pair of values: previous value from storage (value type) and
 223     /// boolean flag (i1 type) with true if success and false otherwise.
 224     std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
 225         RValue Expected, RValue Desired,
 226         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
 227         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
 228         bool IsWeak = false);
 229
 230     /// \brief Emits atomic update.
 231     /// \param AO Atomic ordering.
 232     /// \param UpdateOp Update operation for the current lvalue.
 233     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
 234                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
 235                           bool IsVolatile);
 236     /// \brief Emits atomic update.
 237     /// \param AO Atomic ordering.
 238     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
 239                           bool IsVolatile);
 240
 241     /// Materialize an atomic r-value in atomic-layout memory.
 242     Address materializeRValue(RValue rvalue) const;
 243
 244     /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
 245     /// libcalls.
 246     static AtomicExpr::AtomicOrderingKind
 247     translateAtomicOrdering(const llvm::AtomicOrdering AO);
 248
 249     /// \brief Creates temp alloca for intermediate operations on atomic value.
 250     Address CreateTempAlloca() const;
 251   private:
 252     bool requiresMemSetZero(llvm::Type *type) const;
 253
 254
 255     /// \brief Emits atomic load as a libcall.
 256     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
 257                                llvm::AtomicOrdering AO, bool IsVolatile);
 258     /// \brief Emits atomic load as LLVM instruction.
 259     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
 260     /// \brief Emits atomic compare-and-exchange op as a libcall.
 261     llvm::Value *EmitAtomicCompareExchangeLibcall(
 262         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
 263         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
 264         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
 265     /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
 266     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
 267         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
 268         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
 269         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
 270         bool IsWeak = false);
 271     /// \brief Emit atomic update as libcalls.
 272     void
 273     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
 274                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 275                             bool IsVolatile);
 276     /// \brief Emit atomic update as LLVM instructions.
 277     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
 278                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 279                             bool IsVolatile);
 280     /// \brief Emit atomic update as libcalls.
 281     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
 282                                  bool IsVolatile);
 283     /// \brief Emit atomic update as LLVM instructions.
 284     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
 285                             bool IsVolatile);
 286   };
 287 }
 288
 289 AtomicExpr::AtomicOrderingKind
 290 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
 291   switch (AO) {
 292   case llvm::Unordered:
 293   case llvm::NotAtomic:
 294   case llvm::Monotonic:
 295     return AtomicExpr::AO_ABI_memory_order_relaxed;
 296   case llvm::Acquire:
 297     return AtomicExpr::AO_ABI_memory_order_acquire;
 298   case llvm::Release:
 299     return AtomicExpr::AO_ABI_memory_order_release;
 300   case llvm::AcquireRelease:
 301     return AtomicExpr::AO_ABI_memory_order_acq_rel;
 302   case llvm::SequentiallyConsistent:
 303     return AtomicExpr::AO_ABI_memory_order_seq_cst;
 304   }
 305   llvm_unreachable("Unhandled AtomicOrdering");
 306 }
 307
 308 Address AtomicInfo::CreateTempAlloca() const {
 309   Address TempAlloca = CGF.CreateMemTemp(
 310       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
 311                                                                 : AtomicTy,
 312       getAtomicAlignment(),
 313       "atomic-temp");
 314   // Cast to pointer to value type for bitfields.
 315   if (LVal.isBitField())
 316     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
 317         TempAlloca, getAtomicAddress().getType());
 318   return TempAlloca;
 319 }
 320
 321 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
 322                                 StringRef fnName,
 323                                 QualType resultType,
 324                                 CallArgList &args) {
 325   const CGFunctionInfo &fnInfo =
 326     CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
 327             FunctionType::ExtInfo(), RequiredArgs::All);
 328   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
 329   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
 330   return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
 331 }
 332
 333 /// Does a store of the given IR type modify the full expected width?
 334 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
 335                            uint64_t expectedSize) {
 336   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
 337 }
 338
 339 /// Does the atomic type require memsetting to zero before initialization?
 340 ///
 341 /// The IR type is provided as a way of making certain queries faster.
 342 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
 343   // If the atomic type has size padding, we definitely need a memset.
 344   if (hasPadding()) return true;
 345
 346   // Otherwise, do some simple heuristics to try to avoid it:
 347   switch (getEvaluationKind()) {
 348   // For scalars and complexes, check whether the store size of the
 349   // type uses the full size.
 350   case TEK_Scalar:
 351     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
 352   case TEK_Complex:
 353     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
 354                            AtomicSizeInBits / 2);
 355
 356   // Padding in structs has an undefined bit pattern.  User beware.
 357   case TEK_Aggregate:
 358     return false;
 359   }
 360   llvm_unreachable("bad evaluation kind");
 361 }
 362
 363 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
 364   assert(LVal.isSimple());
 365   llvm::Value *addr = LVal.getPointer();
 366   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
 367     return false;
 368
 369   CGF.Builder.CreateMemSet(
 370       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
 371       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
 372       LVal.getAlignment().getQuantity());
 373   return true;
 374 }
 375
 376 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
 377                               Address Dest, Address Ptr,
 378                               Address Val1, Address Val2,
 379                               uint64_t Size,
 380                               llvm::AtomicOrdering SuccessOrder,
 381                               llvm::AtomicOrdering FailureOrder) {
 382   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
 383   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
 384   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
 385
 386   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
 387       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
 388   Pair->setVolatile(E->isVolatile());
 389   Pair->setWeak(IsWeak);
 390
 391   // Cmp holds the result of the compare-exchange operation: true on success,
 392   // false on failure.
 393   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
 394   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
 395
 396   // This basic block is used to hold the store instruction if the operation
 397   // failed.
 398   llvm::BasicBlock *StoreExpectedBB =
 399       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
 400
 401   // This basic block is the exit point of the operation, we should end up
 402   // here regardless of whether or not the operation succeeded.
 403   llvm::BasicBlock *ContinueBB =
 404       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 405
 406   // Update Expected if Expected isn't equal to Old, otherwise branch to the
 407   // exit point.
 408   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
 409
 410   CGF.Builder.SetInsertPoint(StoreExpectedBB);
 411   // Update the memory at Expected with Old's value.
 412   CGF.Builder.CreateStore(Old, Val1);
 413   // Finally, branch to the exit point.
 414   CGF.Builder.CreateBr(ContinueBB);
 415
 416   CGF.Builder.SetInsertPoint(ContinueBB);
 417   // Update the memory at Dest with Cmp's value.
 418   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
 419 }
 420
 421 /// Given an ordering required on success, emit all possible cmpxchg
 422 /// instructions to cope with the provided (but possibly only dynamically known)
 423 /// FailureOrder.
 424 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
 425                                         bool IsWeak, Address Dest,
 426                                         Address Ptr, Address Val1,
 427                                         Address Val2,
 428                                         llvm::Value *FailureOrderVal,
 429                                         uint64_t Size,
 430                                         llvm::AtomicOrdering SuccessOrder) {
 431   llvm::AtomicOrdering FailureOrder;
 432   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
 433     switch (FO->getSExtValue()) {
 434     default:
 435       FailureOrder = llvm::Monotonic;
 436       break;
 437     case AtomicExpr::AO_ABI_memory_order_consume:
 438     case AtomicExpr::AO_ABI_memory_order_acquire:
 439       FailureOrder = llvm::Acquire;
 440       break;
 441     case AtomicExpr::AO_ABI_memory_order_seq_cst:
 442       FailureOrder = llvm::SequentiallyConsistent;
 443       break;
 444     }
 445     if (FailureOrder >= SuccessOrder) {
 446       // Don't assert on undefined behaviour.
 447       FailureOrder =
 448         llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
 449     }
 450     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size,
 451                       SuccessOrder, FailureOrder);
 452     return;
 453   }
 454
 455   // Create all the relevant BB's
 456   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
 457                    *SeqCstBB = nullptr;
 458   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
 459   if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
 460     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
 461   if (SuccessOrder == llvm::SequentiallyConsistent)
 462     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
 463
 464   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
 465
 466   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
 467
 468   // Emit all the different atomics
 469
 470   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
 471   // doesn't matter unless someone is crazy enough to use something that
 472   // doesn't fold to a constant for the ordering.
 473   CGF.Builder.SetInsertPoint(MonotonicBB);
 474   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 475                     Size, SuccessOrder, llvm::Monotonic);
 476   CGF.Builder.CreateBr(ContBB);
 477
 478   if (AcquireBB) {
 479     CGF.Builder.SetInsertPoint(AcquireBB);
 480     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 481                       Size, SuccessOrder, llvm::Acquire);
 482     CGF.Builder.CreateBr(ContBB);
 483     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
 484                 AcquireBB);
 485     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
 486                 AcquireBB);
 487   }
 488   if (SeqCstBB) {
 489     CGF.Builder.SetInsertPoint(SeqCstBB);
 490     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 491                       Size, SuccessOrder, llvm::SequentiallyConsistent);
 492     CGF.Builder.CreateBr(ContBB);
 493     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
 494                 SeqCstBB);
 495   }
 496
 497   CGF.Builder.SetInsertPoint(ContBB);
 498 }
 499
 500 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
 501                          Address Ptr, Address Val1, Address Val2,
 502                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 503                          uint64_t Size, llvm::AtomicOrdering Order) {
 504   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
 505   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
 506
 507   switch (E->getOp()) {
 508   case AtomicExpr::AO__c11_atomic_init:
 509     llvm_unreachable("Already handled!");
 510
 511   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 512     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 513                                 FailureOrder, Size, Order);
 514     return;
 515   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 516     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 517                                 FailureOrder, Size, Order);
 518     return;
 519   case AtomicExpr::AO__atomic_compare_exchange:
 520   case AtomicExpr::AO__atomic_compare_exchange_n: {
 521     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
 522       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
 523                                   Val1, Val2, FailureOrder, Size, Order);
 524     } else {
 525       // Create all the relevant BB's
 526       llvm::BasicBlock *StrongBB =
 527           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
 528       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
 529       llvm::BasicBlock *ContBB =
 530           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 531
 532       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
 533       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
 534
 535       CGF.Builder.SetInsertPoint(StrongBB);
 536       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 537                                   FailureOrder, Size, Order);
 538       CGF.Builder.CreateBr(ContBB);
 539
 540       CGF.Builder.SetInsertPoint(WeakBB);
 541       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 542                                   FailureOrder, Size, Order);
 543       CGF.Builder.CreateBr(ContBB);
 544
 545       CGF.Builder.SetInsertPoint(ContBB);
 546     }
 547     return;
 548   }
 549   case AtomicExpr::AO__c11_atomic_load:
 550   case AtomicExpr::AO__atomic_load_n:
 551   case AtomicExpr::AO__atomic_load: {
 552     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
 553     Load->setAtomic(Order);
 554     Load->setVolatile(E->isVolatile());
 555     CGF.Builder.CreateStore(Load, Dest);
 556     return;
 557   }
 558
 559   case AtomicExpr::AO__c11_atomic_store:
 560   case AtomicExpr::AO__atomic_store:
 561   case AtomicExpr::AO__atomic_store_n: {
 562     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 563     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
 564     Store->setAtomic(Order);
 565     Store->setVolatile(E->isVolatile());
 566     return;
 567   }
 568
 569   case AtomicExpr::AO__c11_atomic_exchange:
 570   case AtomicExpr::AO__atomic_exchange_n:
 571   case AtomicExpr::AO__atomic_exchange:
 572     Op = llvm::AtomicRMWInst::Xchg;
 573     break;
 574
 575   case AtomicExpr::AO__atomic_add_fetch:
 576     PostOp = llvm::Instruction::Add;
 577     // Fall through.
 578   case AtomicExpr::AO__c11_atomic_fetch_add:
 579   case AtomicExpr::AO__atomic_fetch_add:
 580     Op = llvm::AtomicRMWInst::Add;
 581     break;
 582
 583   case AtomicExpr::AO__atomic_sub_fetch:
 584     PostOp = llvm::Instruction::Sub;
 585     // Fall through.
 586   case AtomicExpr::AO__c11_atomic_fetch_sub:
 587   case AtomicExpr::AO__atomic_fetch_sub:
 588     Op = llvm::AtomicRMWInst::Sub;
 589     break;
 590
 591   case AtomicExpr::AO__atomic_and_fetch:
 592     PostOp = llvm::Instruction::And;
 593     // Fall through.
 594   case AtomicExpr::AO__c11_atomic_fetch_and:
 595   case AtomicExpr::AO__atomic_fetch_and:
 596     Op = llvm::AtomicRMWInst::And;
 597     break;
 598
 599   case AtomicExpr::AO__atomic_or_fetch:
 600     PostOp = llvm::Instruction::Or;
 601     // Fall through.
 602   case AtomicExpr::AO__c11_atomic_fetch_or:
 603   case AtomicExpr::AO__atomic_fetch_or:
 604     Op = llvm::AtomicRMWInst::Or;
 605     break;
 606
 607   case AtomicExpr::AO__atomic_xor_fetch:
 608     PostOp = llvm::Instruction::Xor;
 609     // Fall through.
 610   case AtomicExpr::AO__c11_atomic_fetch_xor:
 611   case AtomicExpr::AO__atomic_fetch_xor:
 612     Op = llvm::AtomicRMWInst::Xor;
 613     break;
 614
 615   case AtomicExpr::AO__atomic_nand_fetch:
 616     PostOp = llvm::Instruction::And; // the NOT is special cased below
 617   // Fall through.
 618   case AtomicExpr::AO__atomic_fetch_nand:
 619     Op = llvm::AtomicRMWInst::Nand;
 620     break;
 621   }
 622
 623   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 624   llvm::AtomicRMWInst *RMWI =
 625       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
 626   RMWI->setVolatile(E->isVolatile());
 627
 628   // For __atomic_*_fetch operations, perform the operation again to
 629   // determine the value which was written.
 630   llvm::Value *Result = RMWI;
 631   if (PostOp)
 632     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
 633   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
 634     Result = CGF.Builder.CreateNot(Result);
 635   CGF.Builder.CreateStore(Result, Dest);
 636 }
 637
 638 // This function emits any expression (scalar, complex, or aggregate)
 639 // into a temporary alloca.
 640 static Address
 641 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
 642   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
 643   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
 644                        /*Init*/ true);
 645   return DeclPtr;
 646 }
 647
 648 static void
 649 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
 650                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
 651                   SourceLocation Loc, CharUnits SizeInChars) {
 652   if (UseOptimizedLibcall) {
 653     // Load value and pass it to the function directly.
 654     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
 655     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
 656     ValTy =
 657         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
 658     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
 659                                                 SizeInBits)->getPointerTo();
 660     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
 661     Val = CGF.EmitLoadOfScalar(Ptr, false,
 662                                CGF.getContext().getPointerType(ValTy),
 663                                Loc);
 664     // Coerce the value into an appropriately sized integer type.
 665     Args.add(RValue::get(Val), ValTy);
 666   } else {
 667     // Non-optimized functions always take a reference.
 668     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
 669                          CGF.getContext().VoidPtrTy);
 670   }
 671 }
 672
 673 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
 674   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
 675   QualType MemTy = AtomicTy;
 676   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
 677     MemTy = AT->getValueType();
 678   CharUnits sizeChars, alignChars;
 679   std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
 680   uint64_t Size = sizeChars.getQuantity();
 681   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
 682   bool UseLibcall = (sizeChars != alignChars ||
 683                      getContext().toBits(sizeChars) > MaxInlineWidthInBits);
 684
 685   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
 686
 687   Address Val1 = Address::invalid();
 688   Address Val2 = Address::invalid();
 689   Address Dest = Address::invalid();
 690   Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
 691
 692   if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
 693     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
 694     EmitAtomicInit(E->getVal1(), lvalue);
 695     return RValue::get(nullptr);
 696   }
 697
 698   llvm::Value *Order = EmitScalarExpr(E->getOrder());
 699
 700   switch (E->getOp()) {
 701   case AtomicExpr::AO__c11_atomic_init:
 702     llvm_unreachable("Already handled above with EmitAtomicInit!");
 703
 704   case AtomicExpr::AO__c11_atomic_load:
 705   case AtomicExpr::AO__atomic_load_n:
 706     break;
 707
 708   case AtomicExpr::AO__atomic_load:
 709     Dest = EmitPointerWithAlignment(E->getVal1());
 710     break;
 711
 712   case AtomicExpr::AO__atomic_store:
 713     Val1 = EmitPointerWithAlignment(E->getVal1());
 714     break;
 715
 716   case AtomicExpr::AO__atomic_exchange:
 717     Val1 = EmitPointerWithAlignment(E->getVal1());
 718     Dest = EmitPointerWithAlignment(E->getVal2());
 719     break;
 720
 721   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 722   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 723   case AtomicExpr::AO__atomic_compare_exchange_n:
 724   case AtomicExpr::AO__atomic_compare_exchange:
 725     Val1 = EmitPointerWithAlignment(E->getVal1());
 726     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 727       Val2 = EmitPointerWithAlignment(E->getVal2());
 728     else
 729       Val2 = EmitValToTemp(*this, E->getVal2());
 730     OrderFail = EmitScalarExpr(E->getOrderFail());
 731     if (E->getNumSubExprs() == 6)
 732       IsWeak = EmitScalarExpr(E->getWeak());
 733     break;
 734
 735   case AtomicExpr::AO__c11_atomic_fetch_add:
 736   case AtomicExpr::AO__c11_atomic_fetch_sub:
 737     if (MemTy->isPointerType()) {
 738       // For pointer arithmetic, we're required to do a bit of math:
 739       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
 740       // ... but only for the C11 builtins. The GNU builtins expect the
 741       // user to multiply by sizeof(T).
 742       QualType Val1Ty = E->getVal1()->getType();
 743       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
 744       CharUnits PointeeIncAmt =
 745           getContext().getTypeSizeInChars(MemTy->getPointeeType());
 746       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
 747       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
 748       Val1 = Temp;
 749       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
 750       break;
 751     }
 752     // Fall through.
 753   case AtomicExpr::AO__atomic_fetch_add:
 754   case AtomicExpr::AO__atomic_fetch_sub:
 755   case AtomicExpr::AO__atomic_add_fetch:
 756   case AtomicExpr::AO__atomic_sub_fetch:
 757   case AtomicExpr::AO__c11_atomic_store:
 758   case AtomicExpr::AO__c11_atomic_exchange:
 759   case AtomicExpr::AO__atomic_store_n:
 760   case AtomicExpr::AO__atomic_exchange_n:
 761   case AtomicExpr::AO__c11_atomic_fetch_and:
 762   case AtomicExpr::AO__c11_atomic_fetch_or:
 763   case AtomicExpr::AO__c11_atomic_fetch_xor:
 764   case AtomicExpr::AO__atomic_fetch_and:
 765   case AtomicExpr::AO__atomic_fetch_or:
 766   case AtomicExpr::AO__atomic_fetch_xor:
 767   case AtomicExpr::AO__atomic_fetch_nand:
 768   case AtomicExpr::AO__atomic_and_fetch:
 769   case AtomicExpr::AO__atomic_or_fetch:
 770   case AtomicExpr::AO__atomic_xor_fetch:
 771   case AtomicExpr::AO__atomic_nand_fetch:
 772     Val1 = EmitValToTemp(*this, E->getVal1());
 773     break;
 774   }
 775
 776   QualType RValTy = E->getType().getUnqualifiedType();
 777
 778   // The inlined atomics only function on iN types, where N is a power of 2. We
 779   // need to make sure (via temporaries if necessary) that all incoming values
 780   // are compatible.
 781   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
 782   AtomicInfo Atomics(*this, AtomicVal);
 783
 784   Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
 785   if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
 786   if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
 787   if (Dest.isValid())
 788     Dest = Atomics.emitCastToAtomicIntPointer(Dest);
 789   else if (E->isCmpXChg())
 790     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
 791   else if (!RValTy->isVoidType())
 792     Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
 793
 794   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
 795   if (UseLibcall) {
 796     bool UseOptimizedLibcall = false;
 797     switch (E->getOp()) {
 798     case AtomicExpr::AO__c11_atomic_init:
 799       llvm_unreachable("Already handled above with EmitAtomicInit!");
 800
 801     case AtomicExpr::AO__c11_atomic_fetch_add:
 802     case AtomicExpr::AO__atomic_fetch_add:
 803     case AtomicExpr::AO__c11_atomic_fetch_and:
 804     case AtomicExpr::AO__atomic_fetch_and:
 805     case AtomicExpr::AO__c11_atomic_fetch_or:
 806     case AtomicExpr::AO__atomic_fetch_or:
 807     case AtomicExpr::AO__atomic_fetch_nand:
 808     case AtomicExpr::AO__c11_atomic_fetch_sub:
 809     case AtomicExpr::AO__atomic_fetch_sub:
 810     case AtomicExpr::AO__c11_atomic_fetch_xor:
 811     case AtomicExpr::AO__atomic_fetch_xor:
 812     case AtomicExpr::AO__atomic_add_fetch:
 813     case AtomicExpr::AO__atomic_and_fetch:
 814     case AtomicExpr::AO__atomic_nand_fetch:
 815     case AtomicExpr::AO__atomic_or_fetch:
 816     case AtomicExpr::AO__atomic_sub_fetch:
 817     case AtomicExpr::AO__atomic_xor_fetch:
 818       // For these, only library calls for certain sizes exist.
 819       UseOptimizedLibcall = true;
 820       break;
 821
 822     case AtomicExpr::AO__c11_atomic_load:
 823     case AtomicExpr::AO__c11_atomic_store:
 824     case AtomicExpr::AO__c11_atomic_exchange:
 825     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 826     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 827     case AtomicExpr::AO__atomic_load_n:
 828     case AtomicExpr::AO__atomic_load:
 829     case AtomicExpr::AO__atomic_store_n:
 830     case AtomicExpr::AO__atomic_store:
 831     case AtomicExpr::AO__atomic_exchange_n:
 832     case AtomicExpr::AO__atomic_exchange:
 833     case AtomicExpr::AO__atomic_compare_exchange_n:
 834     case AtomicExpr::AO__atomic_compare_exchange:
 835       // Only use optimized library calls for sizes for which they exist.
 836       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
 837         UseOptimizedLibcall = true;
 838       break;
 839     }
 840
 841     CallArgList Args;
 842     if (!UseOptimizedLibcall) {
 843       // For non-optimized library calls, the size is the first parameter
 844       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
 845                getContext().getSizeType());
 846     }
 847     // Atomic address is the first or second parameter
 848     Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())),
 849              getContext().VoidPtrTy);
 850
 851     std::string LibCallName;
 852     QualType LoweredMemTy =
 853       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
 854     QualType RetTy;
 855     bool HaveRetTy = false;
 856     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
 857     switch (E->getOp()) {
 858     case AtomicExpr::AO__c11_atomic_init:
 859       llvm_unreachable("Already handled!");
 860
 861     // There is only one libcall for compare an exchange, because there is no
 862     // optimisation benefit possible from a libcall version of a weak compare
 863     // and exchange.
 864     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
 865     //                                void *desired, int success, int failure)
 866     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
 867     //                                  int success, int failure)
 868     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 869     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 870     case AtomicExpr::AO__atomic_compare_exchange:
 871     case AtomicExpr::AO__atomic_compare_exchange_n:
 872       LibCallName = "__atomic_compare_exchange";
 873       RetTy = getContext().BoolTy;
 874       HaveRetTy = true;
 875       Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())),
 876                getContext().VoidPtrTy);
 877       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
 878                         MemTy, E->getExprLoc(), sizeChars);
 879       Args.add(RValue::get(Order), getContext().IntTy);
 880       Order = OrderFail;
 881       break;
 882     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
 883     //                        int order)
 884     // T __atomic_exchange_N(T *mem, T val, int order)
 885     case AtomicExpr::AO__c11_atomic_exchange:
 886     case AtomicExpr::AO__atomic_exchange_n:
 887     case AtomicExpr::AO__atomic_exchange:
 888       LibCallName = "__atomic_exchange";
 889       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 890                         MemTy, E->getExprLoc(), sizeChars);
 891       break;
 892     // void __atomic_store(size_t size, void *mem, void *val, int order)
 893     // void __atomic_store_N(T *mem, T val, int order)
 894     case AtomicExpr::AO__c11_atomic_store:
 895     case AtomicExpr::AO__atomic_store:
 896     case AtomicExpr::AO__atomic_store_n:
 897       LibCallName = "__atomic_store";
 898       RetTy = getContext().VoidTy;
 899       HaveRetTy = true;
 900       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 901                         MemTy, E->getExprLoc(), sizeChars);
 902       break;
 903     // void __atomic_load(size_t size, void *mem, void *return, int order)
 904     // T __atomic_load_N(T *mem, int order)
 905     case AtomicExpr::AO__c11_atomic_load:
 906     case AtomicExpr::AO__atomic_load:
 907     case AtomicExpr::AO__atomic_load_n:
 908       LibCallName = "__atomic_load";
 909       break;
 910     // T __atomic_add_fetch_N(T *mem, T val, int order)
 911     // T __atomic_fetch_add_N(T *mem, T val, int order)
 912     case AtomicExpr::AO__atomic_add_fetch:
 913       PostOp = llvm::Instruction::Add;
 914     // Fall through.
 915     case AtomicExpr::AO__c11_atomic_fetch_add:
 916     case AtomicExpr::AO__atomic_fetch_add:
 917       LibCallName = "__atomic_fetch_add";
 918       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 919                         LoweredMemTy, E->getExprLoc(), sizeChars);
 920       break;
 921     // T __atomic_and_fetch_N(T *mem, T val, int order)
 922     // T __atomic_fetch_and_N(T *mem, T val, int order)
 923     case AtomicExpr::AO__atomic_and_fetch:
 924       PostOp = llvm::Instruction::And;
 925     // Fall through.
 926     case AtomicExpr::AO__c11_atomic_fetch_and:
 927     case AtomicExpr::AO__atomic_fetch_and:
 928       LibCallName = "__atomic_fetch_and";
 929       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 930                         MemTy, E->getExprLoc(), sizeChars);
 931       break;
 932     // T __atomic_or_fetch_N(T *mem, T val, int order)
 933     // T __atomic_fetch_or_N(T *mem, T val, int order)
 934     case AtomicExpr::AO__atomic_or_fetch:
 935       PostOp = llvm::Instruction::Or;
 936     // Fall through.
 937     case AtomicExpr::AO__c11_atomic_fetch_or:
 938     case AtomicExpr::AO__atomic_fetch_or:
 939       LibCallName = "__atomic_fetch_or";
 940       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 941                         MemTy, E->getExprLoc(), sizeChars);
 942       break;
 943     // T __atomic_sub_fetch_N(T *mem, T val, int order)
 944     // T __atomic_fetch_sub_N(T *mem, T val, int order)
 945     case AtomicExpr::AO__atomic_sub_fetch:
 946       PostOp = llvm::Instruction::Sub;
 947     // Fall through.
 948     case AtomicExpr::AO__c11_atomic_fetch_sub:
 949     case AtomicExpr::AO__atomic_fetch_sub:
 950       LibCallName = "__atomic_fetch_sub";
 951       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 952                         LoweredMemTy, E->getExprLoc(), sizeChars);
 953       break;
 954     // T __atomic_xor_fetch_N(T *mem, T val, int order)
 955     // T __atomic_fetch_xor_N(T *mem, T val, int order)
 956     case AtomicExpr::AO__atomic_xor_fetch:
 957       PostOp = llvm::Instruction::Xor;
 958     // Fall through.
 959     case AtomicExpr::AO__c11_atomic_fetch_xor:
 960     case AtomicExpr::AO__atomic_fetch_xor:
 961       LibCallName = "__atomic_fetch_xor";
 962       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 963                         MemTy, E->getExprLoc(), sizeChars);
 964       break;
 965     // T __atomic_nand_fetch_N(T *mem, T val, int order)
 966     // T __atomic_fetch_nand_N(T *mem, T val, int order)
 967     case AtomicExpr::AO__atomic_nand_fetch:
 968       PostOp = llvm::Instruction::And; // the NOT is special cased below
 969     // Fall through.
 970     case AtomicExpr::AO__atomic_fetch_nand:
 971       LibCallName = "__atomic_fetch_nand";
 972       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
 973                         MemTy, E->getExprLoc(), sizeChars);
 974       break;
 975     }
 976
 977     // Optimized functions have the size in their name.
 978     if (UseOptimizedLibcall)
 979       LibCallName += "_" + llvm::utostr(Size);
 980     // By default, assume we return a value of the atomic type.
 981     if (!HaveRetTy) {
 982       if (UseOptimizedLibcall) {
 983         // Value is returned directly.
 984         // The function returns an appropriately sized integer type.
 985         RetTy = getContext().getIntTypeForBitwidth(
 986             getContext().toBits(sizeChars), /*Signed=*/false);
 987       } else {
 988         // Value is returned through parameter before the order.
 989         RetTy = getContext().VoidTy;
 990         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
 991                  getContext().VoidPtrTy);
 992       }
 993     }
 994     // order is always the last parameter
 995     Args.add(RValue::get(Order),
 996              getContext().IntTy);
 997
 998     // PostOp is only needed for the atomic_*_fetch operations, and
 999     // thus is only needed for and implemented in the
1000     // UseOptimizedLibcall codepath.
1001     assert(UseOptimizedLibcall || !PostOp);
1002
1003     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1004     // The value is returned directly from the libcall.
1005     if (E->isCmpXChg())
1006       return Res;
1007
1008     // The value is returned directly for optimized libcalls but the expr
1009     // provided an out-param.
1010     if (UseOptimizedLibcall && Res.getScalarVal()) {
1011       llvm::Value *ResVal = Res.getScalarVal();
1012       if (PostOp) {
1013         llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1014         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1015       }
1016       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1017         ResVal = Builder.CreateNot(ResVal);
1018
1019       Builder.CreateStore(
1020           ResVal,
1021           Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1022     }
1023
1024     if (RValTy->isVoidType())
1025       return RValue::get(nullptr);
1026
1027     return convertTempToRValue(
1028         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1029         RValTy, E->getExprLoc());
1030   }
1031
1032   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1033                  E->getOp() == AtomicExpr::AO__atomic_store ||
1034                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1035   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1036                 E->getOp() == AtomicExpr::AO__atomic_load ||
1037                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1038
1039   if (isa<llvm::ConstantInt>(Order)) {
1040     int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1041     switch (ord) {
1042     case AtomicExpr::AO_ABI_memory_order_relaxed:
1043       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1044                    Size, llvm::Monotonic);
1045       break;
1046     case AtomicExpr::AO_ABI_memory_order_consume:
1047     case AtomicExpr::AO_ABI_memory_order_acquire:
1048       if (IsStore)
1049         break; // Avoid crashing on code with undefined behavior
1050       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1051                    Size, llvm::Acquire);
1052       break;
1053     case AtomicExpr::AO_ABI_memory_order_release:
1054       if (IsLoad)
1055         break; // Avoid crashing on code with undefined behavior
1056       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1057                    Size, llvm::Release);
1058       break;
1059     case AtomicExpr::AO_ABI_memory_order_acq_rel:
1060       if (IsLoad || IsStore)
1061         break; // Avoid crashing on code with undefined behavior
1062       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1063                    Size, llvm::AcquireRelease);
1064       break;
1065     case AtomicExpr::AO_ABI_memory_order_seq_cst:
1066       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1067                    Size, llvm::SequentiallyConsistent);
1068       break;
1069     default: // invalid order
1070       // We should not ever get here normally, but it's hard to
1071       // enforce that in general.
1072       break;
1073     }
1074     if (RValTy->isVoidType())
1075       return RValue::get(nullptr);
1076
1077     return convertTempToRValue(
1078         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1079         RValTy, E->getExprLoc());
1080   }
1081
1082   // Long case, when Order isn't obviously constant.
1083
1084   // Create all the relevant BB's
1085   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1086                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1087                    *SeqCstBB = nullptr;
1088   MonotonicBB = createBasicBlock("monotonic", CurFn);
1089   if (!IsStore)
1090     AcquireBB = createBasicBlock("acquire", CurFn);
1091   if (!IsLoad)
1092     ReleaseBB = createBasicBlock("release", CurFn);
1093   if (!IsLoad && !IsStore)
1094     AcqRelBB = createBasicBlock("acqrel", CurFn);
1095   SeqCstBB = createBasicBlock("seqcst", CurFn);
1096   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1097
1098   // Create the switch for the split
1099   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1100   // doesn't matter unless someone is crazy enough to use something that
1101   // doesn't fold to a constant for the ordering.
1102   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1103   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1104
1105   // Emit all the different atomics
1106   Builder.SetInsertPoint(MonotonicBB);
1107   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1108                Size, llvm::Monotonic);
1109   Builder.CreateBr(ContBB);
1110   if (!IsStore) {
1111     Builder.SetInsertPoint(AcquireBB);
1112     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1113                  Size, llvm::Acquire);
1114     Builder.CreateBr(ContBB);
1115     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1116                 AcquireBB);
1117     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1118                 AcquireBB);
1119   }
1120   if (!IsLoad) {
1121     Builder.SetInsertPoint(ReleaseBB);
1122     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1123                  Size, llvm::Release);
1124     Builder.CreateBr(ContBB);
1125     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1126                 ReleaseBB);
1127   }
1128   if (!IsLoad && !IsStore) {
1129     Builder.SetInsertPoint(AcqRelBB);
1130     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1131                  Size, llvm::AcquireRelease);
1132     Builder.CreateBr(ContBB);
1133     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1134                 AcqRelBB);
1135   }
1136   Builder.SetInsertPoint(SeqCstBB);
1137   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1138                Size, llvm::SequentiallyConsistent);
1139   Builder.CreateBr(ContBB);
1140   SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1141               SeqCstBB);
1142
1143   // Cleanup and return
1144   Builder.SetInsertPoint(ContBB);
1145   if (RValTy->isVoidType())
1146     return RValue::get(nullptr);
1147
1148   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1149   return convertTempToRValue(
1150       Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1151       RValTy, E->getExprLoc());
1152 }
1153
1154 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1155   unsigned addrspace =
1156     cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1157   llvm::IntegerType *ty =
1158     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1159   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1160 }
1161
1162 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1163   llvm::Type *Ty = Addr.getElementType();
1164   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1165   if (SourceSizeInBits != AtomicSizeInBits) {
1166     Address Tmp = CreateTempAlloca();
1167     CGF.Builder.CreateMemCpy(Tmp, Addr,
1168                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1169     Addr = Tmp;
1170   }
1171
1172   return emitCastToAtomicIntPointer(Addr);
1173 }
1174
1175 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1176                                              AggValueSlot resultSlot,
1177                                              SourceLocation loc,
1178                                              bool asValue) const {
1179   if (LVal.isSimple()) {
1180     if (EvaluationKind == TEK_Aggregate)
1181       return resultSlot.asRValue();
1182
1183     // Drill into the padding structure if we have one.
1184     if (hasPadding())
1185       addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1186
1187     // Otherwise, just convert the temporary to an r-value using the
1188     // normal conversion routine.
1189     return CGF.convertTempToRValue(addr, getValueType(), loc);
1190   }
1191   if (!asValue)
1192     // Get RValue from temp memory as atomic for non-simple lvalues
1193     return RValue::get(CGF.Builder.CreateLoad(addr));
1194   if (LVal.isBitField())
1195     return CGF.EmitLoadOfBitfieldLValue(
1196         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1197                              LVal.getAlignmentSource()));
1198   if (LVal.isVectorElt())
1199     return CGF.EmitLoadOfLValue(
1200         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1201                               LVal.getAlignmentSource()), loc);
1202   assert(LVal.isExtVectorElt());
1203   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1204       addr, LVal.getExtVectorElts(), LVal.getType(),
1205       LVal.getAlignmentSource()));
1206 }
1207
1208 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1209                                              AggValueSlot ResultSlot,
1210                                              SourceLocation Loc,
1211                                              bool AsValue) const {
1212   // Try not to in some easy cases.
1213   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1214   if (getEvaluationKind() == TEK_Scalar &&
1215       (((!LVal.isBitField() ||
1216          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1217         !hasPadding()) ||
1218        !AsValue)) {
1219     auto *ValTy = AsValue
1220                       ? CGF.ConvertTypeForMem(ValueTy)
1221                       : getAtomicAddress().getType()->getPointerElementType();
1222     if (ValTy->isIntegerTy()) {
1223       assert(IntVal->getType() == ValTy && "Different integer types.");
1224       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1225     } else if (ValTy->isPointerTy())
1226       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1227     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1228       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1229   }
1230
1231   // Create a temporary.  This needs to be big enough to hold the
1232   // atomic integer.
1233   Address Temp = Address::invalid();
1234   bool TempIsVolatile = false;
1235   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1236     assert(!ResultSlot.isIgnored());
1237     Temp = ResultSlot.getAddress();
1238     TempIsVolatile = ResultSlot.isVolatile();
1239   } else {
1240     Temp = CreateTempAlloca();
1241   }
1242
1243   // Slam the integer into the temporary.
1244   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1245   CGF.Builder.CreateStore(IntVal, CastTemp)
1246       ->setVolatile(TempIsVolatile);
1247
1248   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1249 }
1250
1251 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1252                                        llvm::AtomicOrdering AO, bool) {
1253   // void __atomic_load(size_t size, void *mem, void *return, int order);
1254   CallArgList Args;
1255   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1256   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1257            CGF.getContext().VoidPtrTy);
1258   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1259            CGF.getContext().VoidPtrTy);
1260   Args.add(RValue::get(
1261                llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1262            CGF.getContext().IntTy);
1263   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1264 }
1265
1266 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1267                                           bool IsVolatile) {
1268   // Okay, we're doing this natively.
1269   Address Addr = getAtomicAddressAsAtomicIntPointer();
1270   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1271   Load->setAtomic(AO);
1272
1273   // Other decoration.
1274   if (IsVolatile)
1275     Load->setVolatile(true);
1276   if (LVal.getTBAAInfo())
1277     CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1278   return Load;
1279 }
1280
1281 /// An LValue is a candidate for having its loads and stores be made atomic if
1282 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1283 /// performing such an operation can be performed without a libcall.
1284 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1285   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1286   AtomicInfo AI(*this, LV);
1287   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1288   // An atomic is inline if we don't need to use a libcall.
1289   bool AtomicIsInline = !AI.shouldUseLibcall();
1290   return IsVolatile && AtomicIsInline;
1291 }
1292
1293 /// An type is a candidate for having its loads and stores be made atomic if
1294 /// we are operating under /volatile:ms *and* we know the access is volatile and
1295 /// performing such an operation can be performed without a libcall.
1296 bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
1297                                                     bool IsVolatile) const {
1298   // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1299   bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1300       getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1301   return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1302 }
1303
1304 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1305                                        AggValueSlot Slot) {
1306   llvm::AtomicOrdering AO;
1307   bool IsVolatile = LV.isVolatileQualified();
1308   if (LV.getType()->isAtomicType()) {
1309     AO = llvm::SequentiallyConsistent;
1310   } else {
1311     AO = llvm::Acquire;
1312     IsVolatile = true;
1313   }
1314   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1315 }
1316
1317 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1318                                   bool AsValue, llvm::AtomicOrdering AO,
1319                                   bool IsVolatile) {
1320   // Check whether we should use a library call.
1321   if (shouldUseLibcall()) {
1322     Address TempAddr = Address::invalid();
1323     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1324       assert(getEvaluationKind() == TEK_Aggregate);
1325       TempAddr = ResultSlot.getAddress();
1326     } else
1327       TempAddr = CreateTempAlloca();
1328
1329     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1330
1331     // Okay, turn that back into the original value or whole atomic (for
1332     // non-simple lvalues) type.
1333     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1334   }
1335
1336   // Okay, we're doing this natively.
1337   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1338
1339   // If we're ignoring an aggregate return, don't do anything.
1340   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1341     return RValue::getAggregate(Address::invalid(), false);
1342
1343   // Okay, turn that back into the original value or atomic (for non-simple
1344   // lvalues) type.
1345   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1346 }
1347
1348 /// Emit a load from an l-value of atomic type.  Note that the r-value
1349 /// we produce is an r-value of the atomic *value* type.
1350 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1351                                        llvm::AtomicOrdering AO, bool IsVolatile,
1352                                        AggValueSlot resultSlot) {
1353   AtomicInfo Atomics(*this, src);
1354   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1355                                 IsVolatile);
1356 }
1357
1358 /// Copy an r-value into memory as part of storing to an atomic type.
1359 /// This needs to create a bit-pattern suitable for atomic operations.
1360 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1361   assert(LVal.isSimple());
1362   // If we have an r-value, the rvalue should be of the atomic type,
1363   // which means that the caller is responsible for having zeroed
1364   // any padding.  Just do an aggregate copy of that type.
1365   if (rvalue.isAggregate()) {
1366     CGF.EmitAggregateCopy(getAtomicAddress(),
1367                           rvalue.getAggregateAddress(),
1368                           getAtomicType(),
1369                           (rvalue.isVolatileQualified()
1370                            || LVal.isVolatileQualified()));
1371     return;
1372   }
1373
1374   // Okay, otherwise we're copying stuff.
1375
1376   // Zero out the buffer if necessary.
1377   emitMemSetZeroIfNecessary();
1378
1379   // Drill past the padding if present.
1380   LValue TempLVal = projectValue();
1381
1382   // Okay, store the rvalue in.
1383   if (rvalue.isScalar()) {
1384     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1385   } else {
1386     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1387   }
1388 }
1389
1390
1391 /// Materialize an r-value into memory for the purposes of storing it
1392 /// to an atomic type.
1393 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1394   // Aggregate r-values are already in memory, and EmitAtomicStore
1395   // requires them to be values of the atomic type.
1396   if (rvalue.isAggregate())
1397     return rvalue.getAggregateAddress();
1398
1399   // Otherwise, make a temporary and materialize into it.
1400   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1401   AtomicInfo Atomics(CGF, TempLV);
1402   Atomics.emitCopyIntoMemory(rvalue);
1403   return TempLV.getAddress();
1404 }
1405
1406 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1407   // If we've got a scalar value of the right size, try to avoid going
1408   // through memory.
1409   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1410     llvm::Value *Value = RVal.getScalarVal();
1411     if (isa<llvm::IntegerType>(Value->getType()))
1412       return CGF.EmitToMemory(Value, ValueTy);
1413     else {
1414       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1415           CGF.getLLVMContext(),
1416           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1417       if (isa<llvm::PointerType>(Value->getType()))
1418         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1419       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1420         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1421     }
1422   }
1423   // Otherwise, we need to go through memory.
1424   // Put the r-value in memory.
1425   Address Addr = materializeRValue(RVal);
1426
1427   // Cast the temporary to the atomic int type and pull a value out.
1428   Addr = emitCastToAtomicIntPointer(Addr);
1429   return CGF.Builder.CreateLoad(Addr);
1430 }
1431
1432 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1433     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1434     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1435   // Do the atomic store.
1436   Address Addr = getAtomicAddressAsAtomicIntPointer();
1437   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1438                                                ExpectedVal, DesiredVal,
1439                                                Success, Failure);
1440   // Other decoration.
1441   Inst->setVolatile(LVal.isVolatileQualified());
1442   Inst->setWeak(IsWeak);
1443
1444   // Okay, turn that back into the original value type.
1445   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1446   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1447   return std::make_pair(PreviousVal, SuccessFailureVal);
1448 }
1449
1450 llvm::Value *
1451 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1452                                              llvm::Value *DesiredAddr,
1453                                              llvm::AtomicOrdering Success,
1454                                              llvm::AtomicOrdering Failure) {
1455   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1456   // void *desired, int success, int failure);
1457   CallArgList Args;
1458   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1459   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1460            CGF.getContext().VoidPtrTy);
1461   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1462            CGF.getContext().VoidPtrTy);
1463   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1464            CGF.getContext().VoidPtrTy);
1465   Args.add(RValue::get(llvm::ConstantInt::get(
1466                CGF.IntTy, translateAtomicOrdering(Success))),
1467            CGF.getContext().IntTy);
1468   Args.add(RValue::get(llvm::ConstantInt::get(
1469                CGF.IntTy, translateAtomicOrdering(Failure))),
1470            CGF.getContext().IntTy);
1471   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1472                                               CGF.getContext().BoolTy, Args);
1473
1474   return SuccessFailureRVal.getScalarVal();
1475 }
1476
1477 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1478     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1479     llvm::AtomicOrdering Failure, bool IsWeak) {
1480   if (Failure >= Success)
1481     // Don't assert on undefined behavior.
1482     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1483
1484   // Check whether we should use a library call.
1485   if (shouldUseLibcall()) {
1486     // Produce a source address.
1487     Address ExpectedAddr = materializeRValue(Expected);
1488     Address DesiredAddr = materializeRValue(Desired);
1489     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1490                                                  DesiredAddr.getPointer(),
1491                                                  Success, Failure);
1492     return std::make_pair(
1493         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1494                                   SourceLocation(), /*AsValue=*/false),
1495         Res);
1496   }
1497
1498   // If we've got a scalar value of the right size, try to avoid going
1499   // through memory.
1500   auto *ExpectedVal = convertRValueToInt(Expected);
1501   auto *DesiredVal = convertRValueToInt(Desired);
1502   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1503                                          Failure, IsWeak);
1504   return std::make_pair(
1505       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1506                                 SourceLocation(), /*AsValue=*/false),
1507       Res.second);
1508 }
1509
1510 static void
1511 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1512                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1513                       Address DesiredAddr) {
1514   RValue UpRVal;
1515   LValue AtomicLVal = Atomics.getAtomicLValue();
1516   LValue DesiredLVal;
1517   if (AtomicLVal.isSimple()) {
1518     UpRVal = OldRVal;
1519     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1520   } else {
1521     // Build new lvalue for temp address
1522     Address Ptr = Atomics.materializeRValue(OldRVal);
1523     LValue UpdateLVal;
1524     if (AtomicLVal.isBitField()) {
1525       UpdateLVal =
1526           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1527                                AtomicLVal.getType(),
1528                                AtomicLVal.getAlignmentSource());
1529       DesiredLVal =
1530           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1531                                AtomicLVal.getType(),
1532                                AtomicLVal.getAlignmentSource());
1533     } else if (AtomicLVal.isVectorElt()) {
1534       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1535                                          AtomicLVal.getType(),
1536                                          AtomicLVal.getAlignmentSource());
1537       DesiredLVal = LValue::MakeVectorElt(
1538           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1539           AtomicLVal.getAlignmentSource());
1540     } else {
1541       assert(AtomicLVal.isExtVectorElt());
1542       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1543                                             AtomicLVal.getType(),
1544                                             AtomicLVal.getAlignmentSource());
1545       DesiredLVal = LValue::MakeExtVectorElt(
1546           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1547           AtomicLVal.getAlignmentSource());
1548     }
1549     UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1550     DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1551     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1552   }
1553   // Store new value in the corresponding memory area
1554   RValue NewRVal = UpdateOp(UpRVal);
1555   if (NewRVal.isScalar()) {
1556     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1557   } else {
1558     assert(NewRVal.isComplex());
1559     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1560                            /*isInit=*/false);
1561   }
1562 }
1563
1564 void AtomicInfo::EmitAtomicUpdateLibcall(
1565     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1566     bool IsVolatile) {
1567   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1568
1569   Address ExpectedAddr = CreateTempAlloca();
1570
1571   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1572   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1573   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1574   CGF.EmitBlock(ContBB);
1575   Address DesiredAddr = CreateTempAlloca();
1576   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1577       requiresMemSetZero(getAtomicAddress().getElementType())) {
1578     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1579     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1580   }
1581   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1582                                            AggValueSlot::ignored(),
1583                                            SourceLocation(), /*AsValue=*/false);
1584   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1585   auto *Res =
1586       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1587                                        DesiredAddr.getPointer(),
1588                                        AO, Failure);
1589   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1590   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1591 }
1592
1593 void AtomicInfo::EmitAtomicUpdateOp(
1594     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1595     bool IsVolatile) {
1596   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1597
1598   // Do the atomic load.
1599   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1600   // For non-simple lvalues perform compare-and-swap procedure.
1601   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1602   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1603   auto *CurBB = CGF.Builder.GetInsertBlock();
1604   CGF.EmitBlock(ContBB);
1605   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1606                                              /*NumReservedValues=*/2);
1607   PHI->addIncoming(OldVal, CurBB);
1608   Address NewAtomicAddr = CreateTempAlloca();
1609   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1610   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1611       requiresMemSetZero(getAtomicAddress().getElementType())) {
1612     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1613   }
1614   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1615                                            SourceLocation(), /*AsValue=*/false);
1616   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1617   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1618   // Try to write new value using cmpxchg operation
1619   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1620   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1621   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1622   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1623 }
1624
1625 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1626                                   RValue UpdateRVal, Address DesiredAddr) {
1627   LValue AtomicLVal = Atomics.getAtomicLValue();
1628   LValue DesiredLVal;
1629   // Build new lvalue for temp address
1630   if (AtomicLVal.isBitField()) {
1631     DesiredLVal =
1632         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1633                              AtomicLVal.getType(),
1634                              AtomicLVal.getAlignmentSource());
1635   } else if (AtomicLVal.isVectorElt()) {
1636     DesiredLVal =
1637         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1638                               AtomicLVal.getType(),
1639                               AtomicLVal.getAlignmentSource());
1640   } else {
1641     assert(AtomicLVal.isExtVectorElt());
1642     DesiredLVal = LValue::MakeExtVectorElt(
1643         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1644         AtomicLVal.getAlignmentSource());
1645   }
1646   DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1647   // Store new value in the corresponding memory area
1648   assert(UpdateRVal.isScalar());
1649   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1650 }
1651
1652 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1653                                          RValue UpdateRVal, bool IsVolatile) {
1654   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1655
1656   Address ExpectedAddr = CreateTempAlloca();
1657
1658   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1659   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1660   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1661   CGF.EmitBlock(ContBB);
1662   Address DesiredAddr = CreateTempAlloca();
1663   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1664       requiresMemSetZero(getAtomicAddress().getElementType())) {
1665     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1666     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1667   }
1668   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1669   auto *Res =
1670       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1671                                        DesiredAddr.getPointer(),
1672                                        AO, Failure);
1673   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1674   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1675 }
1676
1677 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1678                                     bool IsVolatile) {
1679   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1680
1681   // Do the atomic load.
1682   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1683   // For non-simple lvalues perform compare-and-swap procedure.
1684   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1685   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1686   auto *CurBB = CGF.Builder.GetInsertBlock();
1687   CGF.EmitBlock(ContBB);
1688   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1689                                              /*NumReservedValues=*/2);
1690   PHI->addIncoming(OldVal, CurBB);
1691   Address NewAtomicAddr = CreateTempAlloca();
1692   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1693   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1694       requiresMemSetZero(getAtomicAddress().getElementType())) {
1695     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1696   }
1697   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1698   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1699   // Try to write new value using cmpxchg operation
1700   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1701   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1702   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1703   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1704 }
1705
1706 void AtomicInfo::EmitAtomicUpdate(
1707     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1708     bool IsVolatile) {
1709   if (shouldUseLibcall()) {
1710     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1711   } else {
1712     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1713   }
1714 }
1715
1716 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1717                                   bool IsVolatile) {
1718   if (shouldUseLibcall()) {
1719     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1720   } else {
1721     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1722   }
1723 }
1724
1725 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1726                                       bool isInit) {
1727   bool IsVolatile = lvalue.isVolatileQualified();
1728   llvm::AtomicOrdering AO;
1729   if (lvalue.getType()->isAtomicType()) {
1730     AO = llvm::SequentiallyConsistent;
1731   } else {
1732     AO = llvm::Release;
1733     IsVolatile = true;
1734   }
1735   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1736 }
1737
1738 /// Emit a store to an l-value of atomic type.
1739 ///
1740 /// Note that the r-value is expected to be an r-value *of the atomic
1741 /// type*; this means that for aggregate r-values, it should include
1742 /// storage for any padding that was necessary.
1743 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1744                                       llvm::AtomicOrdering AO, bool IsVolatile,
1745                                       bool isInit) {
1746   // If this is an aggregate r-value, it should agree in type except
1747   // maybe for address-space qualification.
1748   assert(!rvalue.isAggregate() ||
1749          rvalue.getAggregateAddress().getElementType()
1750            == dest.getAddress().getElementType());
1751
1752   AtomicInfo atomics(*this, dest);
1753   LValue LVal = atomics.getAtomicLValue();
1754
1755   // If this is an initialization, just put the value there normally.
1756   if (LVal.isSimple()) {
1757     if (isInit) {
1758       atomics.emitCopyIntoMemory(rvalue);
1759       return;
1760     }
1761
1762     // Check whether we should use a library call.
1763     if (atomics.shouldUseLibcall()) {
1764       // Produce a source address.
1765       Address srcAddr = atomics.materializeRValue(rvalue);
1766
1767       // void __atomic_store(size_t size, void *mem, void *val, int order)
1768       CallArgList args;
1769       args.add(RValue::get(atomics.getAtomicSizeValue()),
1770                getContext().getSizeType());
1771       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1772                getContext().VoidPtrTy);
1773       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1774                getContext().VoidPtrTy);
1775       args.add(RValue::get(llvm::ConstantInt::get(
1776                    IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1777                getContext().IntTy);
1778       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1779       return;
1780     }
1781
1782     // Okay, we're doing this natively.
1783     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1784
1785     // Do the atomic store.
1786     Address addr =
1787         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1788     intValue = Builder.CreateIntCast(
1789         intValue, addr.getElementType(), /*isSigned=*/false);
1790     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1791
1792     // Initializations don't need to be atomic.
1793     if (!isInit)
1794       store->setAtomic(AO);
1795
1796     // Other decoration.
1797     if (IsVolatile)
1798       store->setVolatile(true);
1799     if (dest.getTBAAInfo())
1800       CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1801     return;
1802   }
1803
1804   // Emit simple atomic update operation.
1805   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1806 }
1807
1808 /// Emit a compare-and-exchange op for atomic type.
1809 ///
1810 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1811     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1812     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1813     AggValueSlot Slot) {
1814   // If this is an aggregate r-value, it should agree in type except
1815   // maybe for address-space qualification.
1816   assert(!Expected.isAggregate() ||
1817          Expected.getAggregateAddress().getElementType() ==
1818              Obj.getAddress().getElementType());
1819   assert(!Desired.isAggregate() ||
1820          Desired.getAggregateAddress().getElementType() ==
1821              Obj.getAddress().getElementType());
1822   AtomicInfo Atomics(*this, Obj);
1823
1824   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1825                                            IsWeak);
1826 }
1827
1828 void CodeGenFunction::EmitAtomicUpdate(
1829     LValue LVal, llvm::AtomicOrdering AO,
1830     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1831   AtomicInfo Atomics(*this, LVal);
1832   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1833 }
1834
1835 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1836   AtomicInfo atomics(*this, dest);
1837
1838   switch (atomics.getEvaluationKind()) {
1839   case TEK_Scalar: {
1840     llvm::Value *value = EmitScalarExpr(init);
1841     atomics.emitCopyIntoMemory(RValue::get(value));
1842     return;
1843   }
1844
1845   case TEK_Complex: {
1846     ComplexPairTy value = EmitComplexExpr(init);
1847     atomics.emitCopyIntoMemory(RValue::getComplex(value));
1848     return;
1849   }
1850
1851   case TEK_Aggregate: {
1852     // Fix up the destination if the initializer isn't an expression
1853     // of atomic type.
1854     bool Zeroed = false;
1855     if (!init->getType()->isAtomicType()) {
1856       Zeroed = atomics.emitMemSetZeroIfNecessary();
1857       dest = atomics.projectValue();
1858     }
1859
1860     // Evaluate the expression directly into the destination.
1861     AggValueSlot slot = AggValueSlot::forLValue(dest,
1862                                         AggValueSlot::IsNotDestructed,
1863                                         AggValueSlot::DoesNotNeedGCBarriers,
1864                                         AggValueSlot::IsNotAliased,
1865                                         Zeroed ? AggValueSlot::IsZeroed :
1866                                                  AggValueSlot::IsNotZeroed);
1867
1868     EmitAggExpr(init, slot);
1869     return;
1870   }
1871   }
1872   llvm_unreachable("bad evaluation kind");
1873 }