contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp

   1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file contains the code for emitting atomic operations.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "CodeGenFunction.h"
  15 #include "CGCall.h"
  16 #include "CGRecordLayout.h"
  17 #include "CodeGenModule.h"
  18 #include "clang/AST/ASTContext.h"
  19 #include "clang/CodeGen/CGFunctionInfo.h"
  20 #include "llvm/ADT/StringExtras.h"
  21 #include "llvm/IR/DataLayout.h"
  22 #include "llvm/IR/Intrinsics.h"
  23 #include "llvm/IR/Operator.h"
  24
  25 using namespace clang;
  26 using namespace CodeGen;
  27
  28 namespace {
  29   class AtomicInfo {
  30     CodeGenFunction &CGF;
  31     QualType AtomicTy;
  32     QualType ValueTy;
  33     uint64_t AtomicSizeInBits;
  34     uint64_t ValueSizeInBits;
  35     CharUnits AtomicAlign;
  36     CharUnits ValueAlign;
  37     CharUnits LValueAlign;
  38     TypeEvaluationKind EvaluationKind;
  39     bool UseLibcall;
  40     LValue LVal;
  41     CGBitFieldInfo BFI;
  42   public:
  43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
  44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
  45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
  46       assert(!lvalue.isGlobalReg());
  47       ASTContext &C = CGF.getContext();
  48       if (lvalue.isSimple()) {
  49         AtomicTy = lvalue.getType();
  50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
  51           ValueTy = ATy->getValueType();
  52         else
  53           ValueTy = AtomicTy;
  54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
  55
  56         uint64_t ValueAlignInBits;
  57         uint64_t AtomicAlignInBits;
  58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
  59         ValueSizeInBits = ValueTI.Width;
  60         ValueAlignInBits = ValueTI.Align;
  61
  62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
  63         AtomicSizeInBits = AtomicTI.Width;
  64         AtomicAlignInBits = AtomicTI.Align;
  65
  66         assert(ValueSizeInBits <= AtomicSizeInBits);
  67         assert(ValueAlignInBits <= AtomicAlignInBits);
  68
  69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
  70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
  71         if (lvalue.getAlignment().isZero())
  72           lvalue.setAlignment(AtomicAlign);
  73
  74         LVal = lvalue;
  75       } else if (lvalue.isBitField()) {
  76         ValueTy = lvalue.getType();
  77         ValueSizeInBits = C.getTypeSize(ValueTy);
  78         auto &OrigBFI = lvalue.getBitFieldInfo();
  79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
  80         AtomicSizeInBits = C.toBits(
  81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
  82                 .RoundUpToAlignment(lvalue.getAlignment()));
  83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
  84         auto OffsetInChars =
  85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
  86             lvalue.getAlignment();
  87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
  88             VoidPtrAddr, OffsetInChars.getQuantity());
  89         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
  90             VoidPtrAddr,
  91             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
  92             "atomic_bitfield_base");
  93         BFI = OrigBFI;
  94         BFI.Offset = Offset;
  95         BFI.StorageSize = AtomicSizeInBits;
  96         BFI.StorageOffset += OffsetInChars;
  97         LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
  98                                     lvalue.getAlignment());
  99         LVal.setTBAAInfo(lvalue.getTBAAInfo());
 100         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
 101         if (AtomicTy.isNull()) {
 102           llvm::APInt Size(
 103               /*numBits=*/32,
 104               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
 105           AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
 106                                             /*IndexTypeQuals=*/0);
 107         }
 108         AtomicAlign = ValueAlign = lvalue.getAlignment();
 109       } else if (lvalue.isVectorElt()) {
 110         ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
 111         ValueSizeInBits = C.getTypeSize(ValueTy);
 112         AtomicTy = lvalue.getType();
 113         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 114         AtomicAlign = ValueAlign = lvalue.getAlignment();
 115         LVal = lvalue;
 116       } else {
 117         assert(lvalue.isExtVectorElt());
 118         ValueTy = lvalue.getType();
 119         ValueSizeInBits = C.getTypeSize(ValueTy);
 120         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
 121             lvalue.getType(), lvalue.getExtVectorAddr()
 122                                   ->getType()
 123                                   ->getPointerElementType()
 124                                   ->getVectorNumElements());
 125         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 126         AtomicAlign = ValueAlign = lvalue.getAlignment();
 127         LVal = lvalue;
 128       }
 129       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
 130           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
 131     }
 132
 133     QualType getAtomicType() const { return AtomicTy; }
 134     QualType getValueType() const { return ValueTy; }
 135     CharUnits getAtomicAlignment() const { return AtomicAlign; }
 136     CharUnits getValueAlignment() const { return ValueAlign; }
 137     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
 138     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
 139     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
 140     bool shouldUseLibcall() const { return UseLibcall; }
 141     const LValue &getAtomicLValue() const { return LVal; }
 142     llvm::Value *getAtomicAddress() const {
 143       if (LVal.isSimple())
 144         return LVal.getAddress();
 145       else if (LVal.isBitField())
 146         return LVal.getBitFieldAddr();
 147       else if (LVal.isVectorElt())
 148         return LVal.getVectorAddr();
 149       assert(LVal.isExtVectorElt());
 150       return LVal.getExtVectorAddr();
 151     }
 152
 153     /// Is the atomic size larger than the underlying value type?
 154     ///
 155     /// Note that the absence of padding does not mean that atomic
 156     /// objects are completely interchangeable with non-atomic
 157     /// objects: we might have promoted the alignment of a type
 158     /// without making it bigger.
 159     bool hasPadding() const {
 160       return (ValueSizeInBits != AtomicSizeInBits);
 161     }
 162
 163     bool emitMemSetZeroIfNecessary() const;
 164
 165     llvm::Value *getAtomicSizeValue() const {
 166       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
 167       return CGF.CGM.getSize(size);
 168     }
 169
 170     /// Cast the given pointer to an integer pointer suitable for
 171     /// atomic operations.
 172     llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
 173
 174     /// Turn an atomic-layout object into an r-value.
 175     RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
 176                                SourceLocation loc, bool AsValue) const;
 177
 178     /// \brief Converts a rvalue to integer value.
 179     llvm::Value *convertRValueToInt(RValue RVal) const;
 180
 181     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
 182                                      AggValueSlot ResultSlot,
 183                                      SourceLocation Loc, bool AsValue) const;
 184
 185     /// Copy an atomic r-value into atomic-layout memory.
 186     void emitCopyIntoMemory(RValue rvalue) const;
 187
 188     /// Project an l-value down to the value field.
 189     LValue projectValue() const {
 190       assert(LVal.isSimple());
 191       llvm::Value *addr = getAtomicAddress();
 192       if (hasPadding())
 193         addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
 194
 195       return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
 196                               CGF.getContext(), LVal.getTBAAInfo());
 197     }
 198
 199     /// \brief Emits atomic load.
 200     /// \returns Loaded value.
 201     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
 202                           bool AsValue, llvm::AtomicOrdering AO,
 203                           bool IsVolatile);
 204
 205     /// \brief Emits atomic compare-and-exchange sequence.
 206     /// \param Expected Expected value.
 207     /// \param Desired Desired value.
 208     /// \param Success Atomic ordering for success operation.
 209     /// \param Failure Atomic ordering for failed operation.
 210     /// \param IsWeak true if atomic operation is weak, false otherwise.
 211     /// \returns Pair of values: previous value from storage (value type) and
 212     /// boolean flag (i1 type) with true if success and false otherwise.
 213     std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
 214         RValue Expected, RValue Desired,
 215         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
 216         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
 217         bool IsWeak = false);
 218
 219     /// \brief Emits atomic update.
 220     /// \param AO Atomic ordering.
 221     /// \param UpdateOp Update operation for the current lvalue.
 222     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
 223                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
 224                           bool IsVolatile);
 225     /// \brief Emits atomic update.
 226     /// \param AO Atomic ordering.
 227     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
 228                           bool IsVolatile);
 229
 230     /// Materialize an atomic r-value in atomic-layout memory.
 231     llvm::Value *materializeRValue(RValue rvalue) const;
 232
 233     /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
 234     /// libcalls.
 235     static AtomicExpr::AtomicOrderingKind
 236     translateAtomicOrdering(const llvm::AtomicOrdering AO);
 237
 238   private:
 239     bool requiresMemSetZero(llvm::Type *type) const;
 240
 241     /// \brief Creates temp alloca for intermediate operations on atomic value.
 242     llvm::Value *CreateTempAlloca() const;
 243
 244     /// \brief Emits atomic load as a libcall.
 245     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
 246                                llvm::AtomicOrdering AO, bool IsVolatile);
 247     /// \brief Emits atomic load as LLVM instruction.
 248     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
 249     /// \brief Emits atomic compare-and-exchange op as a libcall.
 250     llvm::Value *EmitAtomicCompareExchangeLibcall(
 251         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
 252         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
 253         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
 254     /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
 255     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
 256         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
 257         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
 258         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
 259         bool IsWeak = false);
 260     /// \brief Emit atomic update as libcalls.
 261     void
 262     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
 263                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 264                             bool IsVolatile);
 265     /// \brief Emit atomic update as LLVM instructions.
 266     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
 267                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 268                             bool IsVolatile);
 269     /// \brief Emit atomic update as libcalls.
 270     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
 271                                  bool IsVolatile);
 272     /// \brief Emit atomic update as LLVM instructions.
 273     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
 274                             bool IsVolatile);
 275   };
 276 }
 277
 278 AtomicExpr::AtomicOrderingKind
 279 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
 280   switch (AO) {
 281   case llvm::Unordered:
 282   case llvm::NotAtomic:
 283   case llvm::Monotonic:
 284     return AtomicExpr::AO_ABI_memory_order_relaxed;
 285   case llvm::Acquire:
 286     return AtomicExpr::AO_ABI_memory_order_acquire;
 287   case llvm::Release:
 288     return AtomicExpr::AO_ABI_memory_order_release;
 289   case llvm::AcquireRelease:
 290     return AtomicExpr::AO_ABI_memory_order_acq_rel;
 291   case llvm::SequentiallyConsistent:
 292     return AtomicExpr::AO_ABI_memory_order_seq_cst;
 293   }
 294   llvm_unreachable("Unhandled AtomicOrdering");
 295 }
 296
 297 llvm::Value *AtomicInfo::CreateTempAlloca() const {
 298   auto *TempAlloca = CGF.CreateMemTemp(
 299       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
 300                                                                 : AtomicTy,
 301       "atomic-temp");
 302   TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
 303   // Cast to pointer to value type for bitfields.
 304   if (LVal.isBitField())
 305     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
 306         TempAlloca, getAtomicAddress()->getType());
 307   return TempAlloca;
 308 }
 309
 310 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
 311                                 StringRef fnName,
 312                                 QualType resultType,
 313                                 CallArgList &args) {
 314   const CGFunctionInfo &fnInfo =
 315     CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
 316             FunctionType::ExtInfo(), RequiredArgs::All);
 317   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
 318   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
 319   return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
 320 }
 321
 322 /// Does a store of the given IR type modify the full expected width?
 323 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
 324                            uint64_t expectedSize) {
 325   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
 326 }
 327
 328 /// Does the atomic type require memsetting to zero before initialization?
 329 ///
 330 /// The IR type is provided as a way of making certain queries faster.
 331 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
 332   // If the atomic type has size padding, we definitely need a memset.
 333   if (hasPadding()) return true;
 334
 335   // Otherwise, do some simple heuristics to try to avoid it:
 336   switch (getEvaluationKind()) {
 337   // For scalars and complexes, check whether the store size of the
 338   // type uses the full size.
 339   case TEK_Scalar:
 340     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
 341   case TEK_Complex:
 342     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
 343                            AtomicSizeInBits / 2);
 344
 345   // Padding in structs has an undefined bit pattern.  User beware.
 346   case TEK_Aggregate:
 347     return false;
 348   }
 349   llvm_unreachable("bad evaluation kind");
 350 }
 351
 352 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
 353   assert(LVal.isSimple());
 354   llvm::Value *addr = LVal.getAddress();
 355   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
 356     return false;
 357
 358   CGF.Builder.CreateMemSet(
 359       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
 360       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
 361       LVal.getAlignment().getQuantity());
 362   return true;
 363 }
 364
 365 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
 366                               llvm::Value *Dest, llvm::Value *Ptr,
 367                               llvm::Value *Val1, llvm::Value *Val2,
 368                               uint64_t Size, unsigned Align,
 369                               llvm::AtomicOrdering SuccessOrder,
 370                               llvm::AtomicOrdering FailureOrder) {
 371   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
 372   llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
 373   Expected->setAlignment(Align);
 374   llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
 375   Desired->setAlignment(Align);
 376
 377   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
 378       Ptr, Expected, Desired, SuccessOrder, FailureOrder);
 379   Pair->setVolatile(E->isVolatile());
 380   Pair->setWeak(IsWeak);
 381
 382   // Cmp holds the result of the compare-exchange operation: true on success,
 383   // false on failure.
 384   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
 385   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
 386
 387   // This basic block is used to hold the store instruction if the operation
 388   // failed.
 389   llvm::BasicBlock *StoreExpectedBB =
 390       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
 391
 392   // This basic block is the exit point of the operation, we should end up
 393   // here regardless of whether or not the operation succeeded.
 394   llvm::BasicBlock *ContinueBB =
 395       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 396
 397   // Update Expected if Expected isn't equal to Old, otherwise branch to the
 398   // exit point.
 399   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
 400
 401   CGF.Builder.SetInsertPoint(StoreExpectedBB);
 402   // Update the memory at Expected with Old's value.
 403   llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
 404   StoreExpected->setAlignment(Align);
 405   // Finally, branch to the exit point.
 406   CGF.Builder.CreateBr(ContinueBB);
 407
 408   CGF.Builder.SetInsertPoint(ContinueBB);
 409   // Update the memory at Dest with Cmp's value.
 410   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
 411   return;
 412 }
 413
 414 /// Given an ordering required on success, emit all possible cmpxchg
 415 /// instructions to cope with the provided (but possibly only dynamically known)
 416 /// FailureOrder.
 417 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
 418                                         bool IsWeak, llvm::Value *Dest,
 419                                         llvm::Value *Ptr, llvm::Value *Val1,
 420                                         llvm::Value *Val2,
 421                                         llvm::Value *FailureOrderVal,
 422                                         uint64_t Size, unsigned Align,
 423                                         llvm::AtomicOrdering SuccessOrder) {
 424   llvm::AtomicOrdering FailureOrder;
 425   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
 426     switch (FO->getSExtValue()) {
 427     default:
 428       FailureOrder = llvm::Monotonic;
 429       break;
 430     case AtomicExpr::AO_ABI_memory_order_consume:
 431     case AtomicExpr::AO_ABI_memory_order_acquire:
 432       FailureOrder = llvm::Acquire;
 433       break;
 434     case AtomicExpr::AO_ABI_memory_order_seq_cst:
 435       FailureOrder = llvm::SequentiallyConsistent;
 436       break;
 437     }
 438     if (FailureOrder >= SuccessOrder) {
 439       // Don't assert on undefined behaviour.
 440       FailureOrder =
 441         llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
 442     }
 443     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
 444                       SuccessOrder, FailureOrder);
 445     return;
 446   }
 447
 448   // Create all the relevant BB's
 449   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
 450                    *SeqCstBB = nullptr;
 451   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
 452   if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
 453     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
 454   if (SuccessOrder == llvm::SequentiallyConsistent)
 455     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
 456
 457   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
 458
 459   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
 460
 461   // Emit all the different atomics
 462
 463   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
 464   // doesn't matter unless someone is crazy enough to use something that
 465   // doesn't fold to a constant for the ordering.
 466   CGF.Builder.SetInsertPoint(MonotonicBB);
 467   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 468                     Size, Align, SuccessOrder, llvm::Monotonic);
 469   CGF.Builder.CreateBr(ContBB);
 470
 471   if (AcquireBB) {
 472     CGF.Builder.SetInsertPoint(AcquireBB);
 473     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 474                       Size, Align, SuccessOrder, llvm::Acquire);
 475     CGF.Builder.CreateBr(ContBB);
 476     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
 477                 AcquireBB);
 478     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
 479                 AcquireBB);
 480   }
 481   if (SeqCstBB) {
 482     CGF.Builder.SetInsertPoint(SeqCstBB);
 483     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 484                       Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
 485     CGF.Builder.CreateBr(ContBB);
 486     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
 487                 SeqCstBB);
 488   }
 489
 490   CGF.Builder.SetInsertPoint(ContBB);
 491 }
 492
 493 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
 494                          llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
 495                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 496                          uint64_t Size, unsigned Align,
 497                          llvm::AtomicOrdering Order) {
 498   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
 499   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
 500
 501   switch (E->getOp()) {
 502   case AtomicExpr::AO__c11_atomic_init:
 503     llvm_unreachable("Already handled!");
 504
 505   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 506     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 507                                 FailureOrder, Size, Align, Order);
 508     return;
 509   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 510     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 511                                 FailureOrder, Size, Align, Order);
 512     return;
 513   case AtomicExpr::AO__atomic_compare_exchange:
 514   case AtomicExpr::AO__atomic_compare_exchange_n: {
 515     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
 516       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
 517                                   Val1, Val2, FailureOrder, Size, Align, Order);
 518     } else {
 519       // Create all the relevant BB's
 520       llvm::BasicBlock *StrongBB =
 521           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
 522       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
 523       llvm::BasicBlock *ContBB =
 524           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 525
 526       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
 527       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
 528
 529       CGF.Builder.SetInsertPoint(StrongBB);
 530       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 531                                   FailureOrder, Size, Align, Order);
 532       CGF.Builder.CreateBr(ContBB);
 533
 534       CGF.Builder.SetInsertPoint(WeakBB);
 535       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 536                                   FailureOrder, Size, Align, Order);
 537       CGF.Builder.CreateBr(ContBB);
 538
 539       CGF.Builder.SetInsertPoint(ContBB);
 540     }
 541     return;
 542   }
 543   case AtomicExpr::AO__c11_atomic_load:
 544   case AtomicExpr::AO__atomic_load_n:
 545   case AtomicExpr::AO__atomic_load: {
 546     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
 547     Load->setAtomic(Order);
 548     Load->setAlignment(Size);
 549     Load->setVolatile(E->isVolatile());
 550     llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
 551     StoreDest->setAlignment(Align);
 552     return;
 553   }
 554
 555   case AtomicExpr::AO__c11_atomic_store:
 556   case AtomicExpr::AO__atomic_store:
 557   case AtomicExpr::AO__atomic_store_n: {
 558     assert(!Dest && "Store does not return a value");
 559     llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 560     LoadVal1->setAlignment(Align);
 561     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
 562     Store->setAtomic(Order);
 563     Store->setAlignment(Size);
 564     Store->setVolatile(E->isVolatile());
 565     return;
 566   }
 567
 568   case AtomicExpr::AO__c11_atomic_exchange:
 569   case AtomicExpr::AO__atomic_exchange_n:
 570   case AtomicExpr::AO__atomic_exchange:
 571     Op = llvm::AtomicRMWInst::Xchg;
 572     break;
 573
 574   case AtomicExpr::AO__atomic_add_fetch:
 575     PostOp = llvm::Instruction::Add;
 576     // Fall through.
 577   case AtomicExpr::AO__c11_atomic_fetch_add:
 578   case AtomicExpr::AO__atomic_fetch_add:
 579     Op = llvm::AtomicRMWInst::Add;
 580     break;
 581
 582   case AtomicExpr::AO__atomic_sub_fetch:
 583     PostOp = llvm::Instruction::Sub;
 584     // Fall through.
 585   case AtomicExpr::AO__c11_atomic_fetch_sub:
 586   case AtomicExpr::AO__atomic_fetch_sub:
 587     Op = llvm::AtomicRMWInst::Sub;
 588     break;
 589
 590   case AtomicExpr::AO__atomic_and_fetch:
 591     PostOp = llvm::Instruction::And;
 592     // Fall through.
 593   case AtomicExpr::AO__c11_atomic_fetch_and:
 594   case AtomicExpr::AO__atomic_fetch_and:
 595     Op = llvm::AtomicRMWInst::And;
 596     break;
 597
 598   case AtomicExpr::AO__atomic_or_fetch:
 599     PostOp = llvm::Instruction::Or;
 600     // Fall through.
 601   case AtomicExpr::AO__c11_atomic_fetch_or:
 602   case AtomicExpr::AO__atomic_fetch_or:
 603     Op = llvm::AtomicRMWInst::Or;
 604     break;
 605
 606   case AtomicExpr::AO__atomic_xor_fetch:
 607     PostOp = llvm::Instruction::Xor;
 608     // Fall through.
 609   case AtomicExpr::AO__c11_atomic_fetch_xor:
 610   case AtomicExpr::AO__atomic_fetch_xor:
 611     Op = llvm::AtomicRMWInst::Xor;
 612     break;
 613
 614   case AtomicExpr::AO__atomic_nand_fetch:
 615     PostOp = llvm::Instruction::And;
 616     // Fall through.
 617   case AtomicExpr::AO__atomic_fetch_nand:
 618     Op = llvm::AtomicRMWInst::Nand;
 619     break;
 620   }
 621
 622   llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 623   LoadVal1->setAlignment(Align);
 624   llvm::AtomicRMWInst *RMWI =
 625       CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
 626   RMWI->setVolatile(E->isVolatile());
 627
 628   // For __atomic_*_fetch operations, perform the operation again to
 629   // determine the value which was written.
 630   llvm::Value *Result = RMWI;
 631   if (PostOp)
 632     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
 633   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
 634     Result = CGF.Builder.CreateNot(Result);
 635   llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
 636   StoreDest->setAlignment(Align);
 637 }
 638
 639 // This function emits any expression (scalar, complex, or aggregate)
 640 // into a temporary alloca.
 641 static llvm::Value *
 642 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
 643   llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
 644   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
 645                        /*Init*/ true);
 646   return DeclPtr;
 647 }
 648
 649 static void
 650 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
 651                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
 652                   SourceLocation Loc, CharUnits SizeInChars) {
 653   if (UseOptimizedLibcall) {
 654     // Load value and pass it to the function directly.
 655     unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
 656     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
 657     ValTy =
 658         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
 659     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
 660                                                 SizeInBits)->getPointerTo();
 661     Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
 662                                Align, CGF.getContext().getPointerType(ValTy),
 663                                Loc);
 664     // Coerce the value into an appropriately sized integer type.
 665     Args.add(RValue::get(Val), ValTy);
 666   } else {
 667     // Non-optimized functions always take a reference.
 668     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
 669                          CGF.getContext().VoidPtrTy);
 670   }
 671 }
 672
 673 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
 674   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
 675   QualType MemTy = AtomicTy;
 676   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
 677     MemTy = AT->getValueType();
 678   CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
 679   uint64_t Size = sizeChars.getQuantity();
 680   CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
 681   unsigned Align = alignChars.getQuantity();
 682   unsigned MaxInlineWidthInBits =
 683     getTarget().getMaxAtomicInlineWidth();
 684   bool UseLibcall = (Size != Align ||
 685                      getContext().toBits(sizeChars) > MaxInlineWidthInBits);
 686
 687   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
 688               *Val2 = nullptr;
 689   llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
 690
 691   if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
 692     assert(!Dest && "Init does not return a value");
 693     LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
 694     EmitAtomicInit(E->getVal1(), lvalue);
 695     return RValue::get(nullptr);
 696   }
 697
 698   llvm::Value *Order = EmitScalarExpr(E->getOrder());
 699
 700   switch (E->getOp()) {
 701   case AtomicExpr::AO__c11_atomic_init:
 702     llvm_unreachable("Already handled above with EmitAtomicInit!");
 703
 704   case AtomicExpr::AO__c11_atomic_load:
 705   case AtomicExpr::AO__atomic_load_n:
 706     break;
 707
 708   case AtomicExpr::AO__atomic_load:
 709     Dest = EmitScalarExpr(E->getVal1());
 710     break;
 711
 712   case AtomicExpr::AO__atomic_store:
 713     Val1 = EmitScalarExpr(E->getVal1());
 714     break;
 715
 716   case AtomicExpr::AO__atomic_exchange:
 717     Val1 = EmitScalarExpr(E->getVal1());
 718     Dest = EmitScalarExpr(E->getVal2());
 719     break;
 720
 721   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 722   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 723   case AtomicExpr::AO__atomic_compare_exchange_n:
 724   case AtomicExpr::AO__atomic_compare_exchange:
 725     Val1 = EmitScalarExpr(E->getVal1());
 726     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 727       Val2 = EmitScalarExpr(E->getVal2());
 728     else
 729       Val2 = EmitValToTemp(*this, E->getVal2());
 730     OrderFail = EmitScalarExpr(E->getOrderFail());
 731     if (E->getNumSubExprs() == 6)
 732       IsWeak = EmitScalarExpr(E->getWeak());
 733     break;
 734
 735   case AtomicExpr::AO__c11_atomic_fetch_add:
 736   case AtomicExpr::AO__c11_atomic_fetch_sub:
 737     if (MemTy->isPointerType()) {
 738       // For pointer arithmetic, we're required to do a bit of math:
 739       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
 740       // ... but only for the C11 builtins. The GNU builtins expect the
 741       // user to multiply by sizeof(T).
 742       QualType Val1Ty = E->getVal1()->getType();
 743       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
 744       CharUnits PointeeIncAmt =
 745           getContext().getTypeSizeInChars(MemTy->getPointeeType());
 746       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
 747       Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
 748       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
 749       break;
 750     }
 751     // Fall through.
 752   case AtomicExpr::AO__atomic_fetch_add:
 753   case AtomicExpr::AO__atomic_fetch_sub:
 754   case AtomicExpr::AO__atomic_add_fetch:
 755   case AtomicExpr::AO__atomic_sub_fetch:
 756   case AtomicExpr::AO__c11_atomic_store:
 757   case AtomicExpr::AO__c11_atomic_exchange:
 758   case AtomicExpr::AO__atomic_store_n:
 759   case AtomicExpr::AO__atomic_exchange_n:
 760   case AtomicExpr::AO__c11_atomic_fetch_and:
 761   case AtomicExpr::AO__c11_atomic_fetch_or:
 762   case AtomicExpr::AO__c11_atomic_fetch_xor:
 763   case AtomicExpr::AO__atomic_fetch_and:
 764   case AtomicExpr::AO__atomic_fetch_or:
 765   case AtomicExpr::AO__atomic_fetch_xor:
 766   case AtomicExpr::AO__atomic_fetch_nand:
 767   case AtomicExpr::AO__atomic_and_fetch:
 768   case AtomicExpr::AO__atomic_or_fetch:
 769   case AtomicExpr::AO__atomic_xor_fetch:
 770   case AtomicExpr::AO__atomic_nand_fetch:
 771     Val1 = EmitValToTemp(*this, E->getVal1());
 772     break;
 773   }
 774
 775   QualType RValTy = E->getType().getUnqualifiedType();
 776
 777   auto GetDest = [&] {
 778     if (!RValTy->isVoidType() && !Dest) {
 779       Dest = CreateMemTemp(RValTy, ".atomicdst");
 780     }
 781     return Dest;
 782   };
 783
 784   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
 785   if (UseLibcall) {
 786     bool UseOptimizedLibcall = false;
 787     switch (E->getOp()) {
 788     case AtomicExpr::AO__c11_atomic_init:
 789       llvm_unreachable("Already handled above with EmitAtomicInit!");
 790
 791     case AtomicExpr::AO__c11_atomic_fetch_add:
 792     case AtomicExpr::AO__atomic_fetch_add:
 793     case AtomicExpr::AO__c11_atomic_fetch_and:
 794     case AtomicExpr::AO__atomic_fetch_and:
 795     case AtomicExpr::AO__c11_atomic_fetch_or:
 796     case AtomicExpr::AO__atomic_fetch_or:
 797     case AtomicExpr::AO__atomic_fetch_nand:
 798     case AtomicExpr::AO__c11_atomic_fetch_sub:
 799     case AtomicExpr::AO__atomic_fetch_sub:
 800     case AtomicExpr::AO__c11_atomic_fetch_xor:
 801     case AtomicExpr::AO__atomic_fetch_xor:
 802     case AtomicExpr::AO__atomic_add_fetch:
 803     case AtomicExpr::AO__atomic_and_fetch:
 804     case AtomicExpr::AO__atomic_nand_fetch:
 805     case AtomicExpr::AO__atomic_or_fetch:
 806     case AtomicExpr::AO__atomic_sub_fetch:
 807     case AtomicExpr::AO__atomic_xor_fetch:
 808       // For these, only library calls for certain sizes exist.
 809       UseOptimizedLibcall = true;
 810       break;
 811
 812     case AtomicExpr::AO__c11_atomic_load:
 813     case AtomicExpr::AO__c11_atomic_store:
 814     case AtomicExpr::AO__c11_atomic_exchange:
 815     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 816     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 817     case AtomicExpr::AO__atomic_load_n:
 818     case AtomicExpr::AO__atomic_load:
 819     case AtomicExpr::AO__atomic_store_n:
 820     case AtomicExpr::AO__atomic_store:
 821     case AtomicExpr::AO__atomic_exchange_n:
 822     case AtomicExpr::AO__atomic_exchange:
 823     case AtomicExpr::AO__atomic_compare_exchange_n:
 824     case AtomicExpr::AO__atomic_compare_exchange:
 825       // Only use optimized library calls for sizes for which they exist.
 826       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
 827         UseOptimizedLibcall = true;
 828       break;
 829     }
 830
 831     CallArgList Args;
 832     if (!UseOptimizedLibcall) {
 833       // For non-optimized library calls, the size is the first parameter
 834       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
 835                getContext().getSizeType());
 836     }
 837     // Atomic address is the first or second parameter
 838     Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
 839
 840     std::string LibCallName;
 841     QualType LoweredMemTy =
 842       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
 843     QualType RetTy;
 844     bool HaveRetTy = false;
 845     switch (E->getOp()) {
 846     case AtomicExpr::AO__c11_atomic_init:
 847       llvm_unreachable("Already handled!");
 848
 849     // There is only one libcall for compare an exchange, because there is no
 850     // optimisation benefit possible from a libcall version of a weak compare
 851     // and exchange.
 852     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
 853     //                                void *desired, int success, int failure)
 854     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
 855     //                                  int success, int failure)
 856     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 857     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 858     case AtomicExpr::AO__atomic_compare_exchange:
 859     case AtomicExpr::AO__atomic_compare_exchange_n:
 860       LibCallName = "__atomic_compare_exchange";
 861       RetTy = getContext().BoolTy;
 862       HaveRetTy = true;
 863       Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
 864       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
 865                         E->getExprLoc(), sizeChars);
 866       Args.add(RValue::get(Order), getContext().IntTy);
 867       Order = OrderFail;
 868       break;
 869     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
 870     //                        int order)
 871     // T __atomic_exchange_N(T *mem, T val, int order)
 872     case AtomicExpr::AO__c11_atomic_exchange:
 873     case AtomicExpr::AO__atomic_exchange_n:
 874     case AtomicExpr::AO__atomic_exchange:
 875       LibCallName = "__atomic_exchange";
 876       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 877                         E->getExprLoc(), sizeChars);
 878       break;
 879     // void __atomic_store(size_t size, void *mem, void *val, int order)
 880     // void __atomic_store_N(T *mem, T val, int order)
 881     case AtomicExpr::AO__c11_atomic_store:
 882     case AtomicExpr::AO__atomic_store:
 883     case AtomicExpr::AO__atomic_store_n:
 884       LibCallName = "__atomic_store";
 885       RetTy = getContext().VoidTy;
 886       HaveRetTy = true;
 887       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 888                         E->getExprLoc(), sizeChars);
 889       break;
 890     // void __atomic_load(size_t size, void *mem, void *return, int order)
 891     // T __atomic_load_N(T *mem, int order)
 892     case AtomicExpr::AO__c11_atomic_load:
 893     case AtomicExpr::AO__atomic_load:
 894     case AtomicExpr::AO__atomic_load_n:
 895       LibCallName = "__atomic_load";
 896       break;
 897     // T __atomic_fetch_add_N(T *mem, T val, int order)
 898     case AtomicExpr::AO__c11_atomic_fetch_add:
 899     case AtomicExpr::AO__atomic_fetch_add:
 900       LibCallName = "__atomic_fetch_add";
 901       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
 902                         E->getExprLoc(), sizeChars);
 903       break;
 904     // T __atomic_fetch_and_N(T *mem, T val, int order)
 905     case AtomicExpr::AO__c11_atomic_fetch_and:
 906     case AtomicExpr::AO__atomic_fetch_and:
 907       LibCallName = "__atomic_fetch_and";
 908       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 909                         E->getExprLoc(), sizeChars);
 910       break;
 911     // T __atomic_fetch_or_N(T *mem, T val, int order)
 912     case AtomicExpr::AO__c11_atomic_fetch_or:
 913     case AtomicExpr::AO__atomic_fetch_or:
 914       LibCallName = "__atomic_fetch_or";
 915       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 916                         E->getExprLoc(), sizeChars);
 917       break;
 918     // T __atomic_fetch_sub_N(T *mem, T val, int order)
 919     case AtomicExpr::AO__c11_atomic_fetch_sub:
 920     case AtomicExpr::AO__atomic_fetch_sub:
 921       LibCallName = "__atomic_fetch_sub";
 922       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
 923                         E->getExprLoc(), sizeChars);
 924       break;
 925     // T __atomic_fetch_xor_N(T *mem, T val, int order)
 926     case AtomicExpr::AO__c11_atomic_fetch_xor:
 927     case AtomicExpr::AO__atomic_fetch_xor:
 928       LibCallName = "__atomic_fetch_xor";
 929       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 930                         E->getExprLoc(), sizeChars);
 931       break;
 932     // T __atomic_fetch_nand_N(T *mem, T val, int order)
 933     case AtomicExpr::AO__atomic_fetch_nand:
 934       LibCallName = "__atomic_fetch_nand";
 935       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 936                         E->getExprLoc(), sizeChars);
 937       break;
 938
 939     // T __atomic_add_fetch_N(T *mem, T val, int order)
 940     case AtomicExpr::AO__atomic_add_fetch:
 941       LibCallName = "__atomic_add_fetch";
 942       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
 943                         E->getExprLoc(), sizeChars);
 944       break;
 945     // T __atomic_and_fetch_N(T *mem, T val, int order)
 946     case AtomicExpr::AO__atomic_and_fetch:
 947       LibCallName = "__atomic_and_fetch";
 948       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 949                         E->getExprLoc(), sizeChars);
 950       break;
 951     // T __atomic_or_fetch_N(T *mem, T val, int order)
 952     case AtomicExpr::AO__atomic_or_fetch:
 953       LibCallName = "__atomic_or_fetch";
 954       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 955                         E->getExprLoc(), sizeChars);
 956       break;
 957     // T __atomic_sub_fetch_N(T *mem, T val, int order)
 958     case AtomicExpr::AO__atomic_sub_fetch:
 959       LibCallName = "__atomic_sub_fetch";
 960       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
 961                         E->getExprLoc(), sizeChars);
 962       break;
 963     // T __atomic_xor_fetch_N(T *mem, T val, int order)
 964     case AtomicExpr::AO__atomic_xor_fetch:
 965       LibCallName = "__atomic_xor_fetch";
 966       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 967                         E->getExprLoc(), sizeChars);
 968       break;
 969     // T __atomic_nand_fetch_N(T *mem, T val, int order)
 970     case AtomicExpr::AO__atomic_nand_fetch:
 971       LibCallName = "__atomic_nand_fetch";
 972       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 973                         E->getExprLoc(), sizeChars);
 974       break;
 975     }
 976
 977     // Optimized functions have the size in their name.
 978     if (UseOptimizedLibcall)
 979       LibCallName += "_" + llvm::utostr(Size);
 980     // By default, assume we return a value of the atomic type.
 981     if (!HaveRetTy) {
 982       if (UseOptimizedLibcall) {
 983         // Value is returned directly.
 984         // The function returns an appropriately sized integer type.
 985         RetTy = getContext().getIntTypeForBitwidth(
 986             getContext().toBits(sizeChars), /*Signed=*/false);
 987       } else {
 988         // Value is returned through parameter before the order.
 989         RetTy = getContext().VoidTy;
 990         Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
 991       }
 992     }
 993     // order is always the last parameter
 994     Args.add(RValue::get(Order),
 995              getContext().IntTy);
 996
 997     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
 998     // The value is returned directly from the libcall.
 999     if (HaveRetTy && !RetTy->isVoidType())
1000       return Res;
1001     // The value is returned via an explicit out param.
1002     if (RetTy->isVoidType())
1003       return RValue::get(nullptr);
1004     // The value is returned directly for optimized libcalls but the caller is
1005     // expected an out-param.
1006     if (UseOptimizedLibcall) {
1007       llvm::Value *ResVal = Res.getScalarVal();
1008       llvm::StoreInst *StoreDest = Builder.CreateStore(
1009           ResVal,
1010           Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
1011       StoreDest->setAlignment(Align);
1012     }
1013     return convertTempToRValue(Dest, RValTy, E->getExprLoc());
1014   }
1015
1016   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1017                  E->getOp() == AtomicExpr::AO__atomic_store ||
1018                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1019   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1020                 E->getOp() == AtomicExpr::AO__atomic_load ||
1021                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1022
1023   llvm::Type *ITy =
1024       llvm::IntegerType::get(getLLVMContext(), Size * 8);
1025   llvm::Value *OrigDest = GetDest();
1026   Ptr = Builder.CreateBitCast(
1027       Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
1028   if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
1029   if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
1030   if (Dest && !E->isCmpXChg())
1031     Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
1032
1033   if (isa<llvm::ConstantInt>(Order)) {
1034     int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1035     switch (ord) {
1036     case AtomicExpr::AO_ABI_memory_order_relaxed:
1037       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1038                    Size, Align, llvm::Monotonic);
1039       break;
1040     case AtomicExpr::AO_ABI_memory_order_consume:
1041     case AtomicExpr::AO_ABI_memory_order_acquire:
1042       if (IsStore)
1043         break; // Avoid crashing on code with undefined behavior
1044       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1045                    Size, Align, llvm::Acquire);
1046       break;
1047     case AtomicExpr::AO_ABI_memory_order_release:
1048       if (IsLoad)
1049         break; // Avoid crashing on code with undefined behavior
1050       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1051                    Size, Align, llvm::Release);
1052       break;
1053     case AtomicExpr::AO_ABI_memory_order_acq_rel:
1054       if (IsLoad || IsStore)
1055         break; // Avoid crashing on code with undefined behavior
1056       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1057                    Size, Align, llvm::AcquireRelease);
1058       break;
1059     case AtomicExpr::AO_ABI_memory_order_seq_cst:
1060       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1061                    Size, Align, llvm::SequentiallyConsistent);
1062       break;
1063     default: // invalid order
1064       // We should not ever get here normally, but it's hard to
1065       // enforce that in general.
1066       break;
1067     }
1068     if (RValTy->isVoidType())
1069       return RValue::get(nullptr);
1070     return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1071   }
1072
1073   // Long case, when Order isn't obviously constant.
1074
1075   // Create all the relevant BB's
1076   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1077                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1078                    *SeqCstBB = nullptr;
1079   MonotonicBB = createBasicBlock("monotonic", CurFn);
1080   if (!IsStore)
1081     AcquireBB = createBasicBlock("acquire", CurFn);
1082   if (!IsLoad)
1083     ReleaseBB = createBasicBlock("release", CurFn);
1084   if (!IsLoad && !IsStore)
1085     AcqRelBB = createBasicBlock("acqrel", CurFn);
1086   SeqCstBB = createBasicBlock("seqcst", CurFn);
1087   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1088
1089   // Create the switch for the split
1090   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1091   // doesn't matter unless someone is crazy enough to use something that
1092   // doesn't fold to a constant for the ordering.
1093   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1094   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1095
1096   // Emit all the different atomics
1097   Builder.SetInsertPoint(MonotonicBB);
1098   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1099                Size, Align, llvm::Monotonic);
1100   Builder.CreateBr(ContBB);
1101   if (!IsStore) {
1102     Builder.SetInsertPoint(AcquireBB);
1103     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1104                  Size, Align, llvm::Acquire);
1105     Builder.CreateBr(ContBB);
1106     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1107                 AcquireBB);
1108     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1109                 AcquireBB);
1110   }
1111   if (!IsLoad) {
1112     Builder.SetInsertPoint(ReleaseBB);
1113     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1114                  Size, Align, llvm::Release);
1115     Builder.CreateBr(ContBB);
1116     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1117                 ReleaseBB);
1118   }
1119   if (!IsLoad && !IsStore) {
1120     Builder.SetInsertPoint(AcqRelBB);
1121     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1122                  Size, Align, llvm::AcquireRelease);
1123     Builder.CreateBr(ContBB);
1124     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1125                 AcqRelBB);
1126   }
1127   Builder.SetInsertPoint(SeqCstBB);
1128   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1129                Size, Align, llvm::SequentiallyConsistent);
1130   Builder.CreateBr(ContBB);
1131   SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1132               SeqCstBB);
1133
1134   // Cleanup and return
1135   Builder.SetInsertPoint(ContBB);
1136   if (RValTy->isVoidType())
1137     return RValue::get(nullptr);
1138   return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1139 }
1140
1141 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
1142   unsigned addrspace =
1143     cast<llvm::PointerType>(addr->getType())->getAddressSpace();
1144   llvm::IntegerType *ty =
1145     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1146   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1147 }
1148
1149 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
1150                                        AggValueSlot resultSlot,
1151                                        SourceLocation loc, bool AsValue) const {
1152   if (LVal.isSimple()) {
1153     if (EvaluationKind == TEK_Aggregate)
1154       return resultSlot.asRValue();
1155
1156     // Drill into the padding structure if we have one.
1157     if (hasPadding())
1158       addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
1159
1160     // Otherwise, just convert the temporary to an r-value using the
1161     // normal conversion routine.
1162     return CGF.convertTempToRValue(addr, getValueType(), loc);
1163   }
1164   if (!AsValue)
1165     // Get RValue from temp memory as atomic for non-simple lvalues
1166     return RValue::get(
1167         CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
1168   if (LVal.isBitField())
1169     return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
1170         addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
1171   if (LVal.isVectorElt())
1172     return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
1173                                                       LVal.getType(),
1174                                                       LVal.getAlignment()),
1175                                 loc);
1176   assert(LVal.isExtVectorElt());
1177   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1178       addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
1179 }
1180
1181 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1182                                              AggValueSlot ResultSlot,
1183                                              SourceLocation Loc,
1184                                              bool AsValue) const {
1185   // Try not to in some easy cases.
1186   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1187   if (getEvaluationKind() == TEK_Scalar &&
1188       (((!LVal.isBitField() ||
1189          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1190         !hasPadding()) ||
1191        !AsValue)) {
1192     auto *ValTy = AsValue
1193                       ? CGF.ConvertTypeForMem(ValueTy)
1194                       : getAtomicAddress()->getType()->getPointerElementType();
1195     if (ValTy->isIntegerTy()) {
1196       assert(IntVal->getType() == ValTy && "Different integer types.");
1197       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1198     } else if (ValTy->isPointerTy())
1199       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1200     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1201       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1202   }
1203
1204   // Create a temporary.  This needs to be big enough to hold the
1205   // atomic integer.
1206   llvm::Value *Temp;
1207   bool TempIsVolatile = false;
1208   CharUnits TempAlignment;
1209   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1210     assert(!ResultSlot.isIgnored());
1211     Temp = ResultSlot.getAddr();
1212     TempAlignment = getValueAlignment();
1213     TempIsVolatile = ResultSlot.isVolatile();
1214   } else {
1215     Temp = CreateTempAlloca();
1216     TempAlignment = getAtomicAlignment();
1217   }
1218
1219   // Slam the integer into the temporary.
1220   llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
1221   CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
1222       ->setVolatile(TempIsVolatile);
1223
1224   return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
1225 }
1226
1227 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1228                                        llvm::AtomicOrdering AO, bool) {
1229   // void __atomic_load(size_t size, void *mem, void *return, int order);
1230   CallArgList Args;
1231   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1232   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1233            CGF.getContext().VoidPtrTy);
1234   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1235            CGF.getContext().VoidPtrTy);
1236   Args.add(RValue::get(
1237                llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1238            CGF.getContext().IntTy);
1239   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1240 }
1241
1242 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1243                                           bool IsVolatile) {
1244   // Okay, we're doing this natively.
1245   llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1246   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1247   Load->setAtomic(AO);
1248
1249   // Other decoration.
1250   Load->setAlignment(getAtomicAlignment().getQuantity());
1251   if (IsVolatile)
1252     Load->setVolatile(true);
1253   if (LVal.getTBAAInfo())
1254     CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
1255   return Load;
1256 }
1257
1258 /// An LValue is a candidate for having its loads and stores be made atomic if
1259 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1260 /// performing such an operation can be performed without a libcall.
1261 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1262   AtomicInfo AI(*this, LV);
1263   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1264   // An atomic is inline if we don't need to use a libcall.
1265   bool AtomicIsInline = !AI.shouldUseLibcall();
1266   return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1267 }
1268
1269 /// An type is a candidate for having its loads and stores be made atomic if
1270 /// we are operating under /volatile:ms *and* we know the access is volatile and
1271 /// performing such an operation can be performed without a libcall.
1272 bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
1273                                                     bool IsVolatile) const {
1274   // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1275   bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1276       getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1277   return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1278 }
1279
1280 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1281                                        AggValueSlot Slot) {
1282   llvm::AtomicOrdering AO;
1283   bool IsVolatile = LV.isVolatileQualified();
1284   if (LV.getType()->isAtomicType()) {
1285     AO = llvm::SequentiallyConsistent;
1286   } else {
1287     AO = llvm::Acquire;
1288     IsVolatile = true;
1289   }
1290   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1291 }
1292
1293 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1294                                   bool AsValue, llvm::AtomicOrdering AO,
1295                                   bool IsVolatile) {
1296   // Check whether we should use a library call.
1297   if (shouldUseLibcall()) {
1298     llvm::Value *TempAddr;
1299     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1300       assert(getEvaluationKind() == TEK_Aggregate);
1301       TempAddr = ResultSlot.getAddr();
1302     } else
1303       TempAddr = CreateTempAlloca();
1304
1305     EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
1306
1307     // Okay, turn that back into the original value or whole atomic (for
1308     // non-simple lvalues) type.
1309     return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1310   }
1311
1312   // Okay, we're doing this natively.
1313   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1314
1315   // If we're ignoring an aggregate return, don't do anything.
1316   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1317     return RValue::getAggregate(nullptr, false);
1318
1319   // Okay, turn that back into the original value or atomic (for non-simple
1320   // lvalues) type.
1321   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1322 }
1323
1324 /// Emit a load from an l-value of atomic type.  Note that the r-value
1325 /// we produce is an r-value of the atomic *value* type.
1326 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1327                                        llvm::AtomicOrdering AO, bool IsVolatile,
1328                                        AggValueSlot resultSlot) {
1329   AtomicInfo Atomics(*this, src);
1330   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1331                                 IsVolatile);
1332 }
1333
1334 /// Copy an r-value into memory as part of storing to an atomic type.
1335 /// This needs to create a bit-pattern suitable for atomic operations.
1336 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1337   assert(LVal.isSimple());
1338   // If we have an r-value, the rvalue should be of the atomic type,
1339   // which means that the caller is responsible for having zeroed
1340   // any padding.  Just do an aggregate copy of that type.
1341   if (rvalue.isAggregate()) {
1342     CGF.EmitAggregateCopy(getAtomicAddress(),
1343                           rvalue.getAggregateAddr(),
1344                           getAtomicType(),
1345                           (rvalue.isVolatileQualified()
1346                            || LVal.isVolatileQualified()),
1347                           LVal.getAlignment());
1348     return;
1349   }
1350
1351   // Okay, otherwise we're copying stuff.
1352
1353   // Zero out the buffer if necessary.
1354   emitMemSetZeroIfNecessary();
1355
1356   // Drill past the padding if present.
1357   LValue TempLVal = projectValue();
1358
1359   // Okay, store the rvalue in.
1360   if (rvalue.isScalar()) {
1361     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1362   } else {
1363     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1364   }
1365 }
1366
1367
1368 /// Materialize an r-value into memory for the purposes of storing it
1369 /// to an atomic type.
1370 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1371   // Aggregate r-values are already in memory, and EmitAtomicStore
1372   // requires them to be values of the atomic type.
1373   if (rvalue.isAggregate())
1374     return rvalue.getAggregateAddr();
1375
1376   // Otherwise, make a temporary and materialize into it.
1377   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
1378                                      getAtomicAlignment());
1379   AtomicInfo Atomics(CGF, TempLV);
1380   Atomics.emitCopyIntoMemory(rvalue);
1381   return TempLV.getAddress();
1382 }
1383
1384 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1385   // If we've got a scalar value of the right size, try to avoid going
1386   // through memory.
1387   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1388     llvm::Value *Value = RVal.getScalarVal();
1389     if (isa<llvm::IntegerType>(Value->getType()))
1390       return CGF.EmitToMemory(Value, ValueTy);
1391     else {
1392       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1393           CGF.getLLVMContext(),
1394           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1395       if (isa<llvm::PointerType>(Value->getType()))
1396         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1397       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1398         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1399     }
1400   }
1401   // Otherwise, we need to go through memory.
1402   // Put the r-value in memory.
1403   llvm::Value *Addr = materializeRValue(RVal);
1404
1405   // Cast the temporary to the atomic int type and pull a value out.
1406   Addr = emitCastToAtomicIntPointer(Addr);
1407   return CGF.Builder.CreateAlignedLoad(Addr,
1408                                        getAtomicAlignment().getQuantity());
1409 }
1410
1411 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1412     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1413     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1414   // Do the atomic store.
1415   auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1416   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
1417                                                Success, Failure);
1418   // Other decoration.
1419   Inst->setVolatile(LVal.isVolatileQualified());
1420   Inst->setWeak(IsWeak);
1421
1422   // Okay, turn that back into the original value type.
1423   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1424   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1425   return std::make_pair(PreviousVal, SuccessFailureVal);
1426 }
1427
1428 llvm::Value *
1429 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1430                                              llvm::Value *DesiredAddr,
1431                                              llvm::AtomicOrdering Success,
1432                                              llvm::AtomicOrdering Failure) {
1433   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1434   // void *desired, int success, int failure);
1435   CallArgList Args;
1436   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1437   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1438            CGF.getContext().VoidPtrTy);
1439   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1440            CGF.getContext().VoidPtrTy);
1441   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1442            CGF.getContext().VoidPtrTy);
1443   Args.add(RValue::get(llvm::ConstantInt::get(
1444                CGF.IntTy, translateAtomicOrdering(Success))),
1445            CGF.getContext().IntTy);
1446   Args.add(RValue::get(llvm::ConstantInt::get(
1447                CGF.IntTy, translateAtomicOrdering(Failure))),
1448            CGF.getContext().IntTy);
1449   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1450                                               CGF.getContext().BoolTy, Args);
1451
1452   return SuccessFailureRVal.getScalarVal();
1453 }
1454
1455 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1456     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1457     llvm::AtomicOrdering Failure, bool IsWeak) {
1458   if (Failure >= Success)
1459     // Don't assert on undefined behavior.
1460     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1461
1462   // Check whether we should use a library call.
1463   if (shouldUseLibcall()) {
1464     // Produce a source address.
1465     auto *ExpectedAddr = materializeRValue(Expected);
1466     auto *DesiredAddr = materializeRValue(Desired);
1467     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr,
1468                                                  Success, Failure);
1469     return std::make_pair(
1470         convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1471                             SourceLocation(), /*AsValue=*/false),
1472         Res);
1473   }
1474
1475   // If we've got a scalar value of the right size, try to avoid going
1476   // through memory.
1477   auto *ExpectedVal = convertRValueToInt(Expected);
1478   auto *DesiredVal = convertRValueToInt(Desired);
1479   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1480                                          Failure, IsWeak);
1481   return std::make_pair(
1482       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1483                                 SourceLocation(), /*AsValue=*/false),
1484       Res.second);
1485 }
1486
1487 static void
1488 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1489                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1490                       llvm::Value *DesiredAddr) {
1491   llvm::Value *Ptr = nullptr;
1492   LValue UpdateLVal;
1493   RValue UpRVal;
1494   LValue AtomicLVal = Atomics.getAtomicLValue();
1495   LValue DesiredLVal;
1496   if (AtomicLVal.isSimple()) {
1497     UpRVal = OldRVal;
1498     DesiredLVal =
1499         LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(),
1500                          AtomicLVal.getAlignment(), CGF.CGM.getContext());
1501   } else {
1502     // Build new lvalue for temp address
1503     Ptr = Atomics.materializeRValue(OldRVal);
1504     if (AtomicLVal.isBitField()) {
1505       UpdateLVal =
1506           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1507                                AtomicLVal.getType(), AtomicLVal.getAlignment());
1508       DesiredLVal =
1509           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1510                                AtomicLVal.getType(), AtomicLVal.getAlignment());
1511     } else if (AtomicLVal.isVectorElt()) {
1512       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1513                                          AtomicLVal.getType(),
1514                                          AtomicLVal.getAlignment());
1515       DesiredLVal = LValue::MakeVectorElt(
1516           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1517           AtomicLVal.getAlignment());
1518     } else {
1519       assert(AtomicLVal.isExtVectorElt());
1520       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1521                                             AtomicLVal.getType(),
1522                                             AtomicLVal.getAlignment());
1523       DesiredLVal = LValue::MakeExtVectorElt(
1524           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1525           AtomicLVal.getAlignment());
1526     }
1527     UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1528     DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1529     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1530   }
1531   // Store new value in the corresponding memory area
1532   RValue NewRVal = UpdateOp(UpRVal);
1533   if (NewRVal.isScalar()) {
1534     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1535   } else {
1536     assert(NewRVal.isComplex());
1537     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1538                            /*isInit=*/false);
1539   }
1540 }
1541
1542 void AtomicInfo::EmitAtomicUpdateLibcall(
1543     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1544     bool IsVolatile) {
1545   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1546
1547   llvm::Value *ExpectedAddr = CreateTempAlloca();
1548
1549   EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
1550   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1551   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1552   CGF.EmitBlock(ContBB);
1553   auto *DesiredAddr = CreateTempAlloca();
1554   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1555       requiresMemSetZero(
1556           getAtomicAddress()->getType()->getPointerElementType())) {
1557     auto *OldVal = CGF.Builder.CreateAlignedLoad(
1558         ExpectedAddr, getAtomicAlignment().getQuantity());
1559     CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
1560                                    getAtomicAlignment().getQuantity());
1561   }
1562   auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1563                                     SourceLocation(), /*AsValue=*/false);
1564   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1565   auto *Res =
1566       EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
1567   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1568   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1569 }
1570
1571 void AtomicInfo::EmitAtomicUpdateOp(
1572     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1573     bool IsVolatile) {
1574   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1575
1576   // Do the atomic load.
1577   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1578   // For non-simple lvalues perform compare-and-swap procedure.
1579   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1580   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1581   auto *CurBB = CGF.Builder.GetInsertBlock();
1582   CGF.EmitBlock(ContBB);
1583   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1584                                              /*NumReservedValues=*/2);
1585   PHI->addIncoming(OldVal, CurBB);
1586   auto *NewAtomicAddr = CreateTempAlloca();
1587   auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1588   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1589       requiresMemSetZero(
1590           getAtomicAddress()->getType()->getPointerElementType())) {
1591     CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
1592                                    getAtomicAlignment().getQuantity());
1593   }
1594   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1595                                            SourceLocation(), /*AsValue=*/false);
1596   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1597   auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
1598       NewAtomicIntAddr, getAtomicAlignment().getQuantity());
1599   // Try to write new value using cmpxchg operation
1600   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1601   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1602   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1603   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1604 }
1605
1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1607                                   RValue UpdateRVal, llvm::Value *DesiredAddr) {
1608   LValue AtomicLVal = Atomics.getAtomicLValue();
1609   LValue DesiredLVal;
1610   // Build new lvalue for temp address
1611   if (AtomicLVal.isBitField()) {
1612     DesiredLVal =
1613         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1614                              AtomicLVal.getType(), AtomicLVal.getAlignment());
1615   } else if (AtomicLVal.isVectorElt()) {
1616     DesiredLVal =
1617         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1618                               AtomicLVal.getType(), AtomicLVal.getAlignment());
1619   } else {
1620     assert(AtomicLVal.isExtVectorElt());
1621     DesiredLVal = LValue::MakeExtVectorElt(
1622         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1623         AtomicLVal.getAlignment());
1624   }
1625   DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1626   // Store new value in the corresponding memory area
1627   assert(UpdateRVal.isScalar());
1628   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1629 }
1630
1631 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1632                                          RValue UpdateRVal, bool IsVolatile) {
1633   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1634
1635   llvm::Value *ExpectedAddr = CreateTempAlloca();
1636
1637   EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
1638   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1639   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1640   CGF.EmitBlock(ContBB);
1641   auto *DesiredAddr = CreateTempAlloca();
1642   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1643       requiresMemSetZero(
1644           getAtomicAddress()->getType()->getPointerElementType())) {
1645     auto *OldVal = CGF.Builder.CreateAlignedLoad(
1646         ExpectedAddr, getAtomicAlignment().getQuantity());
1647     CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
1648                                    getAtomicAlignment().getQuantity());
1649   }
1650   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1651   auto *Res =
1652       EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
1653   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1654   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1655 }
1656
1657 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1658                                     bool IsVolatile) {
1659   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1660
1661   // Do the atomic load.
1662   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1663   // For non-simple lvalues perform compare-and-swap procedure.
1664   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1665   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1666   auto *CurBB = CGF.Builder.GetInsertBlock();
1667   CGF.EmitBlock(ContBB);
1668   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1669                                              /*NumReservedValues=*/2);
1670   PHI->addIncoming(OldVal, CurBB);
1671   auto *NewAtomicAddr = CreateTempAlloca();
1672   auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1673   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1674       requiresMemSetZero(
1675           getAtomicAddress()->getType()->getPointerElementType())) {
1676     CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
1677                                    getAtomicAlignment().getQuantity());
1678   }
1679   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1680   auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
1681       NewAtomicIntAddr, getAtomicAlignment().getQuantity());
1682   // Try to write new value using cmpxchg operation
1683   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1684   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1685   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1686   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1687 }
1688
1689 void AtomicInfo::EmitAtomicUpdate(
1690     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1691     bool IsVolatile) {
1692   if (shouldUseLibcall()) {
1693     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1694   } else {
1695     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1696   }
1697 }
1698
1699 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1700                                   bool IsVolatile) {
1701   if (shouldUseLibcall()) {
1702     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1703   } else {
1704     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1705   }
1706 }
1707
1708 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1709                                       bool isInit) {
1710   bool IsVolatile = lvalue.isVolatileQualified();
1711   llvm::AtomicOrdering AO;
1712   if (lvalue.getType()->isAtomicType()) {
1713     AO = llvm::SequentiallyConsistent;
1714   } else {
1715     AO = llvm::Release;
1716     IsVolatile = true;
1717   }
1718   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1719 }
1720
1721 /// Emit a store to an l-value of atomic type.
1722 ///
1723 /// Note that the r-value is expected to be an r-value *of the atomic
1724 /// type*; this means that for aggregate r-values, it should include
1725 /// storage for any padding that was necessary.
1726 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1727                                       llvm::AtomicOrdering AO, bool IsVolatile,
1728                                       bool isInit) {
1729   // If this is an aggregate r-value, it should agree in type except
1730   // maybe for address-space qualification.
1731   assert(!rvalue.isAggregate() ||
1732          rvalue.getAggregateAddr()->getType()->getPointerElementType()
1733            == dest.getAddress()->getType()->getPointerElementType());
1734
1735   AtomicInfo atomics(*this, dest);
1736   LValue LVal = atomics.getAtomicLValue();
1737
1738   // If this is an initialization, just put the value there normally.
1739   if (LVal.isSimple()) {
1740     if (isInit) {
1741       atomics.emitCopyIntoMemory(rvalue);
1742       return;
1743     }
1744
1745     // Check whether we should use a library call.
1746     if (atomics.shouldUseLibcall()) {
1747       // Produce a source address.
1748       llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1749
1750       // void __atomic_store(size_t size, void *mem, void *val, int order)
1751       CallArgList args;
1752       args.add(RValue::get(atomics.getAtomicSizeValue()),
1753                getContext().getSizeType());
1754       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
1755                getContext().VoidPtrTy);
1756       args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
1757       args.add(RValue::get(llvm::ConstantInt::get(
1758                    IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1759                getContext().IntTy);
1760       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1761       return;
1762     }
1763
1764     // Okay, we're doing this natively.
1765     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1766
1767     // Do the atomic store.
1768     llvm::Value *addr =
1769         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1770     intValue = Builder.CreateIntCast(
1771         intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
1772     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1773
1774     // Initializations don't need to be atomic.
1775     if (!isInit)
1776       store->setAtomic(AO);
1777
1778     // Other decoration.
1779     store->setAlignment(dest.getAlignment().getQuantity());
1780     if (IsVolatile)
1781       store->setVolatile(true);
1782     if (dest.getTBAAInfo())
1783       CGM.DecorateInstruction(store, dest.getTBAAInfo());
1784     return;
1785   }
1786
1787   // Emit simple atomic update operation.
1788   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1789 }
1790
1791 /// Emit a compare-and-exchange op for atomic type.
1792 ///
1793 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1794     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1795     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1796     AggValueSlot Slot) {
1797   // If this is an aggregate r-value, it should agree in type except
1798   // maybe for address-space qualification.
1799   assert(!Expected.isAggregate() ||
1800          Expected.getAggregateAddr()->getType()->getPointerElementType() ==
1801              Obj.getAddress()->getType()->getPointerElementType());
1802   assert(!Desired.isAggregate() ||
1803          Desired.getAggregateAddr()->getType()->getPointerElementType() ==
1804              Obj.getAddress()->getType()->getPointerElementType());
1805   AtomicInfo Atomics(*this, Obj);
1806
1807   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1808                                            IsWeak);
1809 }
1810
1811 void CodeGenFunction::EmitAtomicUpdate(
1812     LValue LVal, llvm::AtomicOrdering AO,
1813     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1814   AtomicInfo Atomics(*this, LVal);
1815   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1816 }
1817
1818 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1819   AtomicInfo atomics(*this, dest);
1820
1821   switch (atomics.getEvaluationKind()) {
1822   case TEK_Scalar: {
1823     llvm::Value *value = EmitScalarExpr(init);
1824     atomics.emitCopyIntoMemory(RValue::get(value));
1825     return;
1826   }
1827
1828   case TEK_Complex: {
1829     ComplexPairTy value = EmitComplexExpr(init);
1830     atomics.emitCopyIntoMemory(RValue::getComplex(value));
1831     return;
1832   }
1833
1834   case TEK_Aggregate: {
1835     // Fix up the destination if the initializer isn't an expression
1836     // of atomic type.
1837     bool Zeroed = false;
1838     if (!init->getType()->isAtomicType()) {
1839       Zeroed = atomics.emitMemSetZeroIfNecessary();
1840       dest = atomics.projectValue();
1841     }
1842
1843     // Evaluate the expression directly into the destination.
1844     AggValueSlot slot = AggValueSlot::forLValue(dest,
1845                                         AggValueSlot::IsNotDestructed,
1846                                         AggValueSlot::DoesNotNeedGCBarriers,
1847                                         AggValueSlot::IsNotAliased,
1848                                         Zeroed ? AggValueSlot::IsZeroed :
1849                                                  AggValueSlot::IsNotZeroed);
1850
1851     EmitAggExpr(init, slot);
1852     return;
1853   }
1854   }
1855   llvm_unreachable("bad evaluation kind");
1856 }