contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp

   1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file contains the code for emitting atomic operations.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "CodeGenFunction.h"
  15 #include "CGCall.h"
  16 #include "CodeGenModule.h"
  17 #include "clang/AST/ASTContext.h"
  18 #include "clang/CodeGen/CGFunctionInfo.h"
  19 #include "llvm/ADT/StringExtras.h"
  20 #include "llvm/IR/DataLayout.h"
  21 #include "llvm/IR/Intrinsics.h"
  22 #include "llvm/IR/Operator.h"
  23
  24 using namespace clang;
  25 using namespace CodeGen;
  26
  27 namespace {
  28   class AtomicInfo {
  29     CodeGenFunction &CGF;
  30     QualType AtomicTy;
  31     QualType ValueTy;
  32     uint64_t AtomicSizeInBits;
  33     uint64_t ValueSizeInBits;
  34     CharUnits AtomicAlign;
  35     CharUnits ValueAlign;
  36     CharUnits LValueAlign;
  37     TypeEvaluationKind EvaluationKind;
  38     bool UseLibcall;
  39   public:
  40     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) {
  41       assert(lvalue.isSimple());
  42
  43       AtomicTy = lvalue.getType();
  44       ValueTy = AtomicTy->castAs<AtomicType>()->getValueType();
  45       EvaluationKind = CGF.getEvaluationKind(ValueTy);
  46
  47       ASTContext &C = CGF.getContext();
  48
  49       uint64_t valueAlignInBits;
  50       std::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy);
  51
  52       uint64_t atomicAlignInBits;
  53       std::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy);
  54
  55       assert(ValueSizeInBits <= AtomicSizeInBits);
  56       assert(valueAlignInBits <= atomicAlignInBits);
  57
  58       AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits);
  59       ValueAlign = C.toCharUnitsFromBits(valueAlignInBits);
  60       if (lvalue.getAlignment().isZero())
  61         lvalue.setAlignment(AtomicAlign);
  62
  63       UseLibcall =
  64         (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) ||
  65          AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth());
  66     }
  67
  68     QualType getAtomicType() const { return AtomicTy; }
  69     QualType getValueType() const { return ValueTy; }
  70     CharUnits getAtomicAlignment() const { return AtomicAlign; }
  71     CharUnits getValueAlignment() const { return ValueAlign; }
  72     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
  73     uint64_t getValueSizeInBits() const { return AtomicSizeInBits; }
  74     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
  75     bool shouldUseLibcall() const { return UseLibcall; }
  76
  77     /// Is the atomic size larger than the underlying value type?
  78     ///
  79     /// Note that the absence of padding does not mean that atomic
  80     /// objects are completely interchangeable with non-atomic
  81     /// objects: we might have promoted the alignment of a type
  82     /// without making it bigger.
  83     bool hasPadding() const {
  84       return (ValueSizeInBits != AtomicSizeInBits);
  85     }
  86
  87     bool emitMemSetZeroIfNecessary(LValue dest) const;
  88
  89     llvm::Value *getAtomicSizeValue() const {
  90       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
  91       return CGF.CGM.getSize(size);
  92     }
  93
  94     /// Cast the given pointer to an integer pointer suitable for
  95     /// atomic operations.
  96     llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
  97
  98     /// Turn an atomic-layout object into an r-value.
  99     RValue convertTempToRValue(llvm::Value *addr,
 100                                AggValueSlot resultSlot,
 101                                SourceLocation loc) const;
 102
 103     /// Copy an atomic r-value into atomic-layout memory.
 104     void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const;
 105
 106     /// Project an l-value down to the value field.
 107     LValue projectValue(LValue lvalue) const {
 108       llvm::Value *addr = lvalue.getAddress();
 109       if (hasPadding())
 110         addr = CGF.Builder.CreateStructGEP(addr, 0);
 111
 112       return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(),
 113                               CGF.getContext(), lvalue.getTBAAInfo());
 114     }
 115
 116     /// Materialize an atomic r-value in atomic-layout memory.
 117     llvm::Value *materializeRValue(RValue rvalue) const;
 118
 119   private:
 120     bool requiresMemSetZero(llvm::Type *type) const;
 121   };
 122 }
 123
 124 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
 125                                 StringRef fnName,
 126                                 QualType resultType,
 127                                 CallArgList &args) {
 128   const CGFunctionInfo &fnInfo =
 129     CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
 130             FunctionType::ExtInfo(), RequiredArgs::All);
 131   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
 132   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
 133   return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
 134 }
 135
 136 /// Does a store of the given IR type modify the full expected width?
 137 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
 138                            uint64_t expectedSize) {
 139   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
 140 }
 141
 142 /// Does the atomic type require memsetting to zero before initialization?
 143 ///
 144 /// The IR type is provided as a way of making certain queries faster.
 145 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
 146   // If the atomic type has size padding, we definitely need a memset.
 147   if (hasPadding()) return true;
 148
 149   // Otherwise, do some simple heuristics to try to avoid it:
 150   switch (getEvaluationKind()) {
 151   // For scalars and complexes, check whether the store size of the
 152   // type uses the full size.
 153   case TEK_Scalar:
 154     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
 155   case TEK_Complex:
 156     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
 157                            AtomicSizeInBits / 2);
 158
 159   // Padding in structs has an undefined bit pattern.  User beware.
 160   case TEK_Aggregate:
 161     return false;
 162   }
 163   llvm_unreachable("bad evaluation kind");
 164 }
 165
 166 bool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
 167   llvm::Value *addr = dest.getAddress();
 168   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
 169     return false;
 170
 171   CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
 172                            AtomicSizeInBits / 8,
 173                            dest.getAlignment().getQuantity());
 174   return true;
 175 }
 176
 177 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
 178                               llvm::Value *Dest, llvm::Value *Ptr,
 179                               llvm::Value *Val1, llvm::Value *Val2,
 180                               uint64_t Size, unsigned Align,
 181                               llvm::AtomicOrdering SuccessOrder,
 182                               llvm::AtomicOrdering FailureOrder) {
 183   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
 184   llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
 185   Expected->setAlignment(Align);
 186   llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
 187   Desired->setAlignment(Align);
 188
 189   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
 190       Ptr, Expected, Desired, SuccessOrder, FailureOrder);
 191   Pair->setVolatile(E->isVolatile());
 192   Pair->setWeak(IsWeak);
 193
 194   // Cmp holds the result of the compare-exchange operation: true on success,
 195   // false on failure.
 196   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
 197   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
 198
 199   // This basic block is used to hold the store instruction if the operation
 200   // failed.
 201   llvm::BasicBlock *StoreExpectedBB =
 202       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
 203
 204   // This basic block is the exit point of the operation, we should end up
 205   // here regardless of whether or not the operation succeeded.
 206   llvm::BasicBlock *ContinueBB =
 207       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 208
 209   // Update Expected if Expected isn't equal to Old, otherwise branch to the
 210   // exit point.
 211   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
 212
 213   CGF.Builder.SetInsertPoint(StoreExpectedBB);
 214   // Update the memory at Expected with Old's value.
 215   llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
 216   StoreExpected->setAlignment(Align);
 217   // Finally, branch to the exit point.
 218   CGF.Builder.CreateBr(ContinueBB);
 219
 220   CGF.Builder.SetInsertPoint(ContinueBB);
 221   // Update the memory at Dest with Cmp's value.
 222   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
 223   return;
 224 }
 225
 226 /// Given an ordering required on success, emit all possible cmpxchg
 227 /// instructions to cope with the provided (but possibly only dynamically known)
 228 /// FailureOrder.
 229 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
 230                                         bool IsWeak, llvm::Value *Dest,
 231                                         llvm::Value *Ptr, llvm::Value *Val1,
 232                                         llvm::Value *Val2,
 233                                         llvm::Value *FailureOrderVal,
 234                                         uint64_t Size, unsigned Align,
 235                                         llvm::AtomicOrdering SuccessOrder) {
 236   llvm::AtomicOrdering FailureOrder;
 237   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
 238     switch (FO->getSExtValue()) {
 239     default:
 240       FailureOrder = llvm::Monotonic;
 241       break;
 242     case AtomicExpr::AO_ABI_memory_order_consume:
 243     case AtomicExpr::AO_ABI_memory_order_acquire:
 244       FailureOrder = llvm::Acquire;
 245       break;
 246     case AtomicExpr::AO_ABI_memory_order_seq_cst:
 247       FailureOrder = llvm::SequentiallyConsistent;
 248       break;
 249     }
 250     if (FailureOrder >= SuccessOrder) {
 251       // Don't assert on undefined behaviour.
 252       FailureOrder =
 253         llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
 254     }
 255     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
 256                       SuccessOrder, FailureOrder);
 257     return;
 258   }
 259
 260   // Create all the relevant BB's
 261   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
 262                    *SeqCstBB = nullptr;
 263   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
 264   if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
 265     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
 266   if (SuccessOrder == llvm::SequentiallyConsistent)
 267     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
 268
 269   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
 270
 271   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
 272
 273   // Emit all the different atomics
 274
 275   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
 276   // doesn't matter unless someone is crazy enough to use something that
 277   // doesn't fold to a constant for the ordering.
 278   CGF.Builder.SetInsertPoint(MonotonicBB);
 279   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 280                     Size, Align, SuccessOrder, llvm::Monotonic);
 281   CGF.Builder.CreateBr(ContBB);
 282
 283   if (AcquireBB) {
 284     CGF.Builder.SetInsertPoint(AcquireBB);
 285     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 286                       Size, Align, SuccessOrder, llvm::Acquire);
 287     CGF.Builder.CreateBr(ContBB);
 288     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
 289                 AcquireBB);
 290     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
 291                 AcquireBB);
 292   }
 293   if (SeqCstBB) {
 294     CGF.Builder.SetInsertPoint(SeqCstBB);
 295     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 296                       Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
 297     CGF.Builder.CreateBr(ContBB);
 298     SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
 299                 SeqCstBB);
 300   }
 301
 302   CGF.Builder.SetInsertPoint(ContBB);
 303 }
 304
 305 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
 306                          llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
 307                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 308                          uint64_t Size, unsigned Align,
 309                          llvm::AtomicOrdering Order) {
 310   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
 311   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
 312
 313   switch (E->getOp()) {
 314   case AtomicExpr::AO__c11_atomic_init:
 315     llvm_unreachable("Already handled!");
 316
 317   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 318     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 319                                 FailureOrder, Size, Align, Order);
 320     return;
 321   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 322     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 323                                 FailureOrder, Size, Align, Order);
 324     return;
 325   case AtomicExpr::AO__atomic_compare_exchange:
 326   case AtomicExpr::AO__atomic_compare_exchange_n: {
 327     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
 328       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
 329                                   Val1, Val2, FailureOrder, Size, Align, Order);
 330     } else {
 331       // Create all the relevant BB's
 332       llvm::BasicBlock *StrongBB =
 333           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
 334       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
 335       llvm::BasicBlock *ContBB =
 336           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 337
 338       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
 339       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
 340
 341       CGF.Builder.SetInsertPoint(StrongBB);
 342       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 343                                   FailureOrder, Size, Align, Order);
 344       CGF.Builder.CreateBr(ContBB);
 345
 346       CGF.Builder.SetInsertPoint(WeakBB);
 347       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 348                                   FailureOrder, Size, Align, Order);
 349       CGF.Builder.CreateBr(ContBB);
 350
 351       CGF.Builder.SetInsertPoint(ContBB);
 352     }
 353     return;
 354   }
 355   case AtomicExpr::AO__c11_atomic_load:
 356   case AtomicExpr::AO__atomic_load_n:
 357   case AtomicExpr::AO__atomic_load: {
 358     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
 359     Load->setAtomic(Order);
 360     Load->setAlignment(Size);
 361     Load->setVolatile(E->isVolatile());
 362     llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
 363     StoreDest->setAlignment(Align);
 364     return;
 365   }
 366
 367   case AtomicExpr::AO__c11_atomic_store:
 368   case AtomicExpr::AO__atomic_store:
 369   case AtomicExpr::AO__atomic_store_n: {
 370     assert(!Dest && "Store does not return a value");
 371     llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 372     LoadVal1->setAlignment(Align);
 373     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
 374     Store->setAtomic(Order);
 375     Store->setAlignment(Size);
 376     Store->setVolatile(E->isVolatile());
 377     return;
 378   }
 379
 380   case AtomicExpr::AO__c11_atomic_exchange:
 381   case AtomicExpr::AO__atomic_exchange_n:
 382   case AtomicExpr::AO__atomic_exchange:
 383     Op = llvm::AtomicRMWInst::Xchg;
 384     break;
 385
 386   case AtomicExpr::AO__atomic_add_fetch:
 387     PostOp = llvm::Instruction::Add;
 388     // Fall through.
 389   case AtomicExpr::AO__c11_atomic_fetch_add:
 390   case AtomicExpr::AO__atomic_fetch_add:
 391     Op = llvm::AtomicRMWInst::Add;
 392     break;
 393
 394   case AtomicExpr::AO__atomic_sub_fetch:
 395     PostOp = llvm::Instruction::Sub;
 396     // Fall through.
 397   case AtomicExpr::AO__c11_atomic_fetch_sub:
 398   case AtomicExpr::AO__atomic_fetch_sub:
 399     Op = llvm::AtomicRMWInst::Sub;
 400     break;
 401
 402   case AtomicExpr::AO__atomic_and_fetch:
 403     PostOp = llvm::Instruction::And;
 404     // Fall through.
 405   case AtomicExpr::AO__c11_atomic_fetch_and:
 406   case AtomicExpr::AO__atomic_fetch_and:
 407     Op = llvm::AtomicRMWInst::And;
 408     break;
 409
 410   case AtomicExpr::AO__atomic_or_fetch:
 411     PostOp = llvm::Instruction::Or;
 412     // Fall through.
 413   case AtomicExpr::AO__c11_atomic_fetch_or:
 414   case AtomicExpr::AO__atomic_fetch_or:
 415     Op = llvm::AtomicRMWInst::Or;
 416     break;
 417
 418   case AtomicExpr::AO__atomic_xor_fetch:
 419     PostOp = llvm::Instruction::Xor;
 420     // Fall through.
 421   case AtomicExpr::AO__c11_atomic_fetch_xor:
 422   case AtomicExpr::AO__atomic_fetch_xor:
 423     Op = llvm::AtomicRMWInst::Xor;
 424     break;
 425
 426   case AtomicExpr::AO__atomic_nand_fetch:
 427     PostOp = llvm::Instruction::And;
 428     // Fall through.
 429   case AtomicExpr::AO__atomic_fetch_nand:
 430     Op = llvm::AtomicRMWInst::Nand;
 431     break;
 432   }
 433
 434   llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 435   LoadVal1->setAlignment(Align);
 436   llvm::AtomicRMWInst *RMWI =
 437       CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
 438   RMWI->setVolatile(E->isVolatile());
 439
 440   // For __atomic_*_fetch operations, perform the operation again to
 441   // determine the value which was written.
 442   llvm::Value *Result = RMWI;
 443   if (PostOp)
 444     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
 445   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
 446     Result = CGF.Builder.CreateNot(Result);
 447   llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
 448   StoreDest->setAlignment(Align);
 449 }
 450
 451 // This function emits any expression (scalar, complex, or aggregate)
 452 // into a temporary alloca.
 453 static llvm::Value *
 454 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
 455   llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
 456   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
 457                        /*Init*/ true);
 458   return DeclPtr;
 459 }
 460
 461 static void
 462 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
 463                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
 464                   SourceLocation Loc) {
 465   if (UseOptimizedLibcall) {
 466     // Load value and pass it to the function directly.
 467     unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
 468     Val = CGF.EmitLoadOfScalar(Val, false, Align, ValTy, Loc);
 469     Args.add(RValue::get(Val), ValTy);
 470   } else {
 471     // Non-optimized functions always take a reference.
 472     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
 473                          CGF.getContext().VoidPtrTy);
 474   }
 475 }
 476
 477 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
 478   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
 479   QualType MemTy = AtomicTy;
 480   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
 481     MemTy = AT->getValueType();
 482   CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
 483   uint64_t Size = sizeChars.getQuantity();
 484   CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
 485   unsigned Align = alignChars.getQuantity();
 486   unsigned MaxInlineWidthInBits =
 487     getTarget().getMaxAtomicInlineWidth();
 488   bool UseLibcall = (Size != Align ||
 489                      getContext().toBits(sizeChars) > MaxInlineWidthInBits);
 490
 491   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
 492               *Val2 = nullptr;
 493   llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
 494
 495   if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
 496     assert(!Dest && "Init does not return a value");
 497     LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
 498     EmitAtomicInit(E->getVal1(), lvalue);
 499     return RValue::get(nullptr);
 500   }
 501
 502   llvm::Value *Order = EmitScalarExpr(E->getOrder());
 503
 504   switch (E->getOp()) {
 505   case AtomicExpr::AO__c11_atomic_init:
 506     llvm_unreachable("Already handled!");
 507
 508   case AtomicExpr::AO__c11_atomic_load:
 509   case AtomicExpr::AO__atomic_load_n:
 510     break;
 511
 512   case AtomicExpr::AO__atomic_load:
 513     Dest = EmitScalarExpr(E->getVal1());
 514     break;
 515
 516   case AtomicExpr::AO__atomic_store:
 517     Val1 = EmitScalarExpr(E->getVal1());
 518     break;
 519
 520   case AtomicExpr::AO__atomic_exchange:
 521     Val1 = EmitScalarExpr(E->getVal1());
 522     Dest = EmitScalarExpr(E->getVal2());
 523     break;
 524
 525   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 526   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 527   case AtomicExpr::AO__atomic_compare_exchange_n:
 528   case AtomicExpr::AO__atomic_compare_exchange:
 529     Val1 = EmitScalarExpr(E->getVal1());
 530     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 531       Val2 = EmitScalarExpr(E->getVal2());
 532     else
 533       Val2 = EmitValToTemp(*this, E->getVal2());
 534     OrderFail = EmitScalarExpr(E->getOrderFail());
 535     if (E->getNumSubExprs() == 6)
 536       IsWeak = EmitScalarExpr(E->getWeak());
 537     break;
 538
 539   case AtomicExpr::AO__c11_atomic_fetch_add:
 540   case AtomicExpr::AO__c11_atomic_fetch_sub:
 541     if (MemTy->isPointerType()) {
 542       // For pointer arithmetic, we're required to do a bit of math:
 543       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
 544       // ... but only for the C11 builtins. The GNU builtins expect the
 545       // user to multiply by sizeof(T).
 546       QualType Val1Ty = E->getVal1()->getType();
 547       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
 548       CharUnits PointeeIncAmt =
 549           getContext().getTypeSizeInChars(MemTy->getPointeeType());
 550       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
 551       Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
 552       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
 553       break;
 554     }
 555     // Fall through.
 556   case AtomicExpr::AO__atomic_fetch_add:
 557   case AtomicExpr::AO__atomic_fetch_sub:
 558   case AtomicExpr::AO__atomic_add_fetch:
 559   case AtomicExpr::AO__atomic_sub_fetch:
 560   case AtomicExpr::AO__c11_atomic_store:
 561   case AtomicExpr::AO__c11_atomic_exchange:
 562   case AtomicExpr::AO__atomic_store_n:
 563   case AtomicExpr::AO__atomic_exchange_n:
 564   case AtomicExpr::AO__c11_atomic_fetch_and:
 565   case AtomicExpr::AO__c11_atomic_fetch_or:
 566   case AtomicExpr::AO__c11_atomic_fetch_xor:
 567   case AtomicExpr::AO__atomic_fetch_and:
 568   case AtomicExpr::AO__atomic_fetch_or:
 569   case AtomicExpr::AO__atomic_fetch_xor:
 570   case AtomicExpr::AO__atomic_fetch_nand:
 571   case AtomicExpr::AO__atomic_and_fetch:
 572   case AtomicExpr::AO__atomic_or_fetch:
 573   case AtomicExpr::AO__atomic_xor_fetch:
 574   case AtomicExpr::AO__atomic_nand_fetch:
 575     Val1 = EmitValToTemp(*this, E->getVal1());
 576     break;
 577   }
 578
 579   if (!E->getType()->isVoidType() && !Dest)
 580     Dest = CreateMemTemp(E->getType(), ".atomicdst");
 581
 582   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
 583   if (UseLibcall) {
 584     bool UseOptimizedLibcall = false;
 585     switch (E->getOp()) {
 586     case AtomicExpr::AO__c11_atomic_fetch_add:
 587     case AtomicExpr::AO__atomic_fetch_add:
 588     case AtomicExpr::AO__c11_atomic_fetch_and:
 589     case AtomicExpr::AO__atomic_fetch_and:
 590     case AtomicExpr::AO__c11_atomic_fetch_or:
 591     case AtomicExpr::AO__atomic_fetch_or:
 592     case AtomicExpr::AO__c11_atomic_fetch_sub:
 593     case AtomicExpr::AO__atomic_fetch_sub:
 594     case AtomicExpr::AO__c11_atomic_fetch_xor:
 595     case AtomicExpr::AO__atomic_fetch_xor:
 596       // For these, only library calls for certain sizes exist.
 597       UseOptimizedLibcall = true;
 598       break;
 599     default:
 600       // Only use optimized library calls for sizes for which they exist.
 601       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
 602         UseOptimizedLibcall = true;
 603       break;
 604     }
 605
 606     CallArgList Args;
 607     if (!UseOptimizedLibcall) {
 608       // For non-optimized library calls, the size is the first parameter
 609       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
 610                getContext().getSizeType());
 611     }
 612     // Atomic address is the first or second parameter
 613     Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
 614
 615     std::string LibCallName;
 616     QualType LoweredMemTy =
 617       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
 618     QualType RetTy;
 619     bool HaveRetTy = false;
 620     switch (E->getOp()) {
 621     // There is only one libcall for compare an exchange, because there is no
 622     // optimisation benefit possible from a libcall version of a weak compare
 623     // and exchange.
 624     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
 625     //                                void *desired, int success, int failure)
 626     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
 627     //                                  int success, int failure)
 628     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 629     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 630     case AtomicExpr::AO__atomic_compare_exchange:
 631     case AtomicExpr::AO__atomic_compare_exchange_n:
 632       LibCallName = "__atomic_compare_exchange";
 633       RetTy = getContext().BoolTy;
 634       HaveRetTy = true;
 635       Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
 636       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
 637                         E->getExprLoc());
 638       Args.add(RValue::get(Order), getContext().IntTy);
 639       Order = OrderFail;
 640       break;
 641     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
 642     //                        int order)
 643     // T __atomic_exchange_N(T *mem, T val, int order)
 644     case AtomicExpr::AO__c11_atomic_exchange:
 645     case AtomicExpr::AO__atomic_exchange_n:
 646     case AtomicExpr::AO__atomic_exchange:
 647       LibCallName = "__atomic_exchange";
 648       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 649                         E->getExprLoc());
 650       break;
 651     // void __atomic_store(size_t size, void *mem, void *val, int order)
 652     // void __atomic_store_N(T *mem, T val, int order)
 653     case AtomicExpr::AO__c11_atomic_store:
 654     case AtomicExpr::AO__atomic_store:
 655     case AtomicExpr::AO__atomic_store_n:
 656       LibCallName = "__atomic_store";
 657       RetTy = getContext().VoidTy;
 658       HaveRetTy = true;
 659       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 660                         E->getExprLoc());
 661       break;
 662     // void __atomic_load(size_t size, void *mem, void *return, int order)
 663     // T __atomic_load_N(T *mem, int order)
 664     case AtomicExpr::AO__c11_atomic_load:
 665     case AtomicExpr::AO__atomic_load:
 666     case AtomicExpr::AO__atomic_load_n:
 667       LibCallName = "__atomic_load";
 668       break;
 669     // T __atomic_fetch_add_N(T *mem, T val, int order)
 670     case AtomicExpr::AO__c11_atomic_fetch_add:
 671     case AtomicExpr::AO__atomic_fetch_add:
 672       LibCallName = "__atomic_fetch_add";
 673       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
 674                         E->getExprLoc());
 675       break;
 676     // T __atomic_fetch_and_N(T *mem, T val, int order)
 677     case AtomicExpr::AO__c11_atomic_fetch_and:
 678     case AtomicExpr::AO__atomic_fetch_and:
 679       LibCallName = "__atomic_fetch_and";
 680       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 681                         E->getExprLoc());
 682       break;
 683     // T __atomic_fetch_or_N(T *mem, T val, int order)
 684     case AtomicExpr::AO__c11_atomic_fetch_or:
 685     case AtomicExpr::AO__atomic_fetch_or:
 686       LibCallName = "__atomic_fetch_or";
 687       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 688                         E->getExprLoc());
 689       break;
 690     // T __atomic_fetch_sub_N(T *mem, T val, int order)
 691     case AtomicExpr::AO__c11_atomic_fetch_sub:
 692     case AtomicExpr::AO__atomic_fetch_sub:
 693       LibCallName = "__atomic_fetch_sub";
 694       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
 695                         E->getExprLoc());
 696       break;
 697     // T __atomic_fetch_xor_N(T *mem, T val, int order)
 698     case AtomicExpr::AO__c11_atomic_fetch_xor:
 699     case AtomicExpr::AO__atomic_fetch_xor:
 700       LibCallName = "__atomic_fetch_xor";
 701       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
 702                         E->getExprLoc());
 703       break;
 704     default: return EmitUnsupportedRValue(E, "atomic library call");
 705     }
 706
 707     // Optimized functions have the size in their name.
 708     if (UseOptimizedLibcall)
 709       LibCallName += "_" + llvm::utostr(Size);
 710     // By default, assume we return a value of the atomic type.
 711     if (!HaveRetTy) {
 712       if (UseOptimizedLibcall) {
 713         // Value is returned directly.
 714         RetTy = MemTy;
 715       } else {
 716         // Value is returned through parameter before the order.
 717         RetTy = getContext().VoidTy;
 718         Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
 719                  getContext().VoidPtrTy);
 720       }
 721     }
 722     // order is always the last parameter
 723     Args.add(RValue::get(Order),
 724              getContext().IntTy);
 725
 726     const CGFunctionInfo &FuncInfo =
 727         CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args,
 728             FunctionType::ExtInfo(), RequiredArgs::All);
 729     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
 730     llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
 731     RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
 732     if (!RetTy->isVoidType())
 733       return Res;
 734     if (E->getType()->isVoidType())
 735       return RValue::get(nullptr);
 736     return convertTempToRValue(Dest, E->getType(), E->getExprLoc());
 737   }
 738
 739   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
 740                  E->getOp() == AtomicExpr::AO__atomic_store ||
 741                  E->getOp() == AtomicExpr::AO__atomic_store_n;
 742   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
 743                 E->getOp() == AtomicExpr::AO__atomic_load ||
 744                 E->getOp() == AtomicExpr::AO__atomic_load_n;
 745
 746   llvm::Type *IPtrTy =
 747       llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
 748   llvm::Value *OrigDest = Dest;
 749   Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
 750   if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
 751   if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
 752   if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
 753
 754   if (isa<llvm::ConstantInt>(Order)) {
 755     int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
 756     switch (ord) {
 757     case AtomicExpr::AO_ABI_memory_order_relaxed:
 758       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 759                    Size, Align, llvm::Monotonic);
 760       break;
 761     case AtomicExpr::AO_ABI_memory_order_consume:
 762     case AtomicExpr::AO_ABI_memory_order_acquire:
 763       if (IsStore)
 764         break; // Avoid crashing on code with undefined behavior
 765       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 766                    Size, Align, llvm::Acquire);
 767       break;
 768     case AtomicExpr::AO_ABI_memory_order_release:
 769       if (IsLoad)
 770         break; // Avoid crashing on code with undefined behavior
 771       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 772                    Size, Align, llvm::Release);
 773       break;
 774     case AtomicExpr::AO_ABI_memory_order_acq_rel:
 775       if (IsLoad || IsStore)
 776         break; // Avoid crashing on code with undefined behavior
 777       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 778                    Size, Align, llvm::AcquireRelease);
 779       break;
 780     case AtomicExpr::AO_ABI_memory_order_seq_cst:
 781       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 782                    Size, Align, llvm::SequentiallyConsistent);
 783       break;
 784     default: // invalid order
 785       // We should not ever get here normally, but it's hard to
 786       // enforce that in general.
 787       break;
 788     }
 789     if (E->getType()->isVoidType())
 790       return RValue::get(nullptr);
 791     return convertTempToRValue(OrigDest, E->getType(), E->getExprLoc());
 792   }
 793
 794   // Long case, when Order isn't obviously constant.
 795
 796   // Create all the relevant BB's
 797   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
 798                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
 799                    *SeqCstBB = nullptr;
 800   MonotonicBB = createBasicBlock("monotonic", CurFn);
 801   if (!IsStore)
 802     AcquireBB = createBasicBlock("acquire", CurFn);
 803   if (!IsLoad)
 804     ReleaseBB = createBasicBlock("release", CurFn);
 805   if (!IsLoad && !IsStore)
 806     AcqRelBB = createBasicBlock("acqrel", CurFn);
 807   SeqCstBB = createBasicBlock("seqcst", CurFn);
 808   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
 809
 810   // Create the switch for the split
 811   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
 812   // doesn't matter unless someone is crazy enough to use something that
 813   // doesn't fold to a constant for the ordering.
 814   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
 815   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
 816
 817   // Emit all the different atomics
 818   Builder.SetInsertPoint(MonotonicBB);
 819   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 820                Size, Align, llvm::Monotonic);
 821   Builder.CreateBr(ContBB);
 822   if (!IsStore) {
 823     Builder.SetInsertPoint(AcquireBB);
 824     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 825                  Size, Align, llvm::Acquire);
 826     Builder.CreateBr(ContBB);
 827     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
 828                 AcquireBB);
 829     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
 830                 AcquireBB);
 831   }
 832   if (!IsLoad) {
 833     Builder.SetInsertPoint(ReleaseBB);
 834     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 835                  Size, Align, llvm::Release);
 836     Builder.CreateBr(ContBB);
 837     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
 838                 ReleaseBB);
 839   }
 840   if (!IsLoad && !IsStore) {
 841     Builder.SetInsertPoint(AcqRelBB);
 842     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 843                  Size, Align, llvm::AcquireRelease);
 844     Builder.CreateBr(ContBB);
 845     SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
 846                 AcqRelBB);
 847   }
 848   Builder.SetInsertPoint(SeqCstBB);
 849   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
 850                Size, Align, llvm::SequentiallyConsistent);
 851   Builder.CreateBr(ContBB);
 852   SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
 853               SeqCstBB);
 854
 855   // Cleanup and return
 856   Builder.SetInsertPoint(ContBB);
 857   if (E->getType()->isVoidType())
 858     return RValue::get(nullptr);
 859   return convertTempToRValue(OrigDest, E->getType(), E->getExprLoc());
 860 }
 861
 862 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
 863   unsigned addrspace =
 864     cast<llvm::PointerType>(addr->getType())->getAddressSpace();
 865   llvm::IntegerType *ty =
 866     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
 867   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
 868 }
 869
 870 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
 871                                        AggValueSlot resultSlot,
 872                                        SourceLocation loc) const {
 873   if (EvaluationKind == TEK_Aggregate)
 874     return resultSlot.asRValue();
 875
 876   // Drill into the padding structure if we have one.
 877   if (hasPadding())
 878     addr = CGF.Builder.CreateStructGEP(addr, 0);
 879
 880   // Otherwise, just convert the temporary to an r-value using the
 881   // normal conversion routine.
 882   return CGF.convertTempToRValue(addr, getValueType(), loc);
 883 }
 884
 885 /// Emit a load from an l-value of atomic type.  Note that the r-value
 886 /// we produce is an r-value of the atomic *value* type.
 887 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
 888                                        AggValueSlot resultSlot) {
 889   AtomicInfo atomics(*this, src);
 890
 891   // Check whether we should use a library call.
 892   if (atomics.shouldUseLibcall()) {
 893     llvm::Value *tempAddr;
 894     if (!resultSlot.isIgnored()) {
 895       assert(atomics.getEvaluationKind() == TEK_Aggregate);
 896       tempAddr = resultSlot.getAddr();
 897     } else {
 898       tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
 899     }
 900
 901     // void __atomic_load(size_t size, void *mem, void *return, int order);
 902     CallArgList args;
 903     args.add(RValue::get(atomics.getAtomicSizeValue()),
 904              getContext().getSizeType());
 905     args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
 906              getContext().VoidPtrTy);
 907     args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
 908              getContext().VoidPtrTy);
 909     args.add(RValue::get(llvm::ConstantInt::get(
 910                  IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
 911              getContext().IntTy);
 912     emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
 913
 914     // Produce the r-value.
 915     return atomics.convertTempToRValue(tempAddr, resultSlot, loc);
 916   }
 917
 918   // Okay, we're doing this natively.
 919   llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
 920   llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
 921   load->setAtomic(llvm::SequentiallyConsistent);
 922
 923   // Other decoration.
 924   load->setAlignment(src.getAlignment().getQuantity());
 925   if (src.isVolatileQualified())
 926     load->setVolatile(true);
 927   if (src.getTBAAInfo())
 928     CGM.DecorateInstruction(load, src.getTBAAInfo());
 929
 930   // Okay, turn that back into the original value type.
 931   QualType valueType = atomics.getValueType();
 932   llvm::Value *result = load;
 933
 934   // If we're ignoring an aggregate return, don't do anything.
 935   if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
 936     return RValue::getAggregate(nullptr, false);
 937
 938   // The easiest way to do this this is to go through memory, but we
 939   // try not to in some easy cases.
 940   if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) {
 941     llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
 942     if (isa<llvm::IntegerType>(resultTy)) {
 943       assert(result->getType() == resultTy);
 944       result = EmitFromMemory(result, valueType);
 945     } else if (isa<llvm::PointerType>(resultTy)) {
 946       result = Builder.CreateIntToPtr(result, resultTy);
 947     } else {
 948       result = Builder.CreateBitCast(result, resultTy);
 949     }
 950     return RValue::get(result);
 951   }
 952
 953   // Create a temporary.  This needs to be big enough to hold the
 954   // atomic integer.
 955   llvm::Value *temp;
 956   bool tempIsVolatile = false;
 957   CharUnits tempAlignment;
 958   if (atomics.getEvaluationKind() == TEK_Aggregate) {
 959     assert(!resultSlot.isIgnored());
 960     temp = resultSlot.getAddr();
 961     tempAlignment = atomics.getValueAlignment();
 962     tempIsVolatile = resultSlot.isVolatile();
 963   } else {
 964     temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
 965     tempAlignment = atomics.getAtomicAlignment();
 966   }
 967
 968   // Slam the integer into the temporary.
 969   llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
 970   Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity())
 971     ->setVolatile(tempIsVolatile);
 972
 973   return atomics.convertTempToRValue(temp, resultSlot, loc);
 974 }
 975
 976
 977
 978 /// Copy an r-value into memory as part of storing to an atomic type.
 979 /// This needs to create a bit-pattern suitable for atomic operations.
 980 void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
 981   // If we have an r-value, the rvalue should be of the atomic type,
 982   // which means that the caller is responsible for having zeroed
 983   // any padding.  Just do an aggregate copy of that type.
 984   if (rvalue.isAggregate()) {
 985     CGF.EmitAggregateCopy(dest.getAddress(),
 986                           rvalue.getAggregateAddr(),
 987                           getAtomicType(),
 988                           (rvalue.isVolatileQualified()
 989                            || dest.isVolatileQualified()),
 990                           dest.getAlignment());
 991     return;
 992   }
 993
 994   // Okay, otherwise we're copying stuff.
 995
 996   // Zero out the buffer if necessary.
 997   emitMemSetZeroIfNecessary(dest);
 998
 999   // Drill past the padding if present.
1000   dest = projectValue(dest);
1001
1002   // Okay, store the rvalue in.
1003   if (rvalue.isScalar()) {
1004     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
1005   } else {
1006     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
1007   }
1008 }
1009
1010
1011 /// Materialize an r-value into memory for the purposes of storing it
1012 /// to an atomic type.
1013 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1014   // Aggregate r-values are already in memory, and EmitAtomicStore
1015   // requires them to be values of the atomic type.
1016   if (rvalue.isAggregate())
1017     return rvalue.getAggregateAddr();
1018
1019   // Otherwise, make a temporary and materialize into it.
1020   llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
1021   LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment());
1022   emitCopyIntoMemory(rvalue, tempLV);
1023   return temp;
1024 }
1025
1026 /// Emit a store to an l-value of atomic type.
1027 ///
1028 /// Note that the r-value is expected to be an r-value *of the atomic
1029 /// type*; this means that for aggregate r-values, it should include
1030 /// storage for any padding that was necessary.
1031 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
1032   // If this is an aggregate r-value, it should agree in type except
1033   // maybe for address-space qualification.
1034   assert(!rvalue.isAggregate() ||
1035          rvalue.getAggregateAddr()->getType()->getPointerElementType()
1036            == dest.getAddress()->getType()->getPointerElementType());
1037
1038   AtomicInfo atomics(*this, dest);
1039
1040   // If this is an initialization, just put the value there normally.
1041   if (isInit) {
1042     atomics.emitCopyIntoMemory(rvalue, dest);
1043     return;
1044   }
1045
1046   // Check whether we should use a library call.
1047   if (atomics.shouldUseLibcall()) {
1048     // Produce a source address.
1049     llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1050
1051     // void __atomic_store(size_t size, void *mem, void *val, int order)
1052     CallArgList args;
1053     args.add(RValue::get(atomics.getAtomicSizeValue()),
1054              getContext().getSizeType());
1055     args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
1056              getContext().VoidPtrTy);
1057     args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
1058              getContext().VoidPtrTy);
1059     args.add(RValue::get(llvm::ConstantInt::get(
1060                  IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
1061              getContext().IntTy);
1062     emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1063     return;
1064   }
1065
1066   // Okay, we're doing this natively.
1067   llvm::Value *intValue;
1068
1069   // If we've got a scalar value of the right size, try to avoid going
1070   // through memory.
1071   if (rvalue.isScalar() && !atomics.hasPadding()) {
1072     llvm::Value *value = rvalue.getScalarVal();
1073     if (isa<llvm::IntegerType>(value->getType())) {
1074       intValue = value;
1075     } else {
1076       llvm::IntegerType *inputIntTy =
1077         llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits());
1078       if (isa<llvm::PointerType>(value->getType())) {
1079         intValue = Builder.CreatePtrToInt(value, inputIntTy);
1080       } else {
1081         intValue = Builder.CreateBitCast(value, inputIntTy);
1082       }
1083     }
1084
1085   // Otherwise, we need to go through memory.
1086   } else {
1087     // Put the r-value in memory.
1088     llvm::Value *addr = atomics.materializeRValue(rvalue);
1089
1090     // Cast the temporary to the atomic int type and pull a value out.
1091     addr = atomics.emitCastToAtomicIntPointer(addr);
1092     intValue = Builder.CreateAlignedLoad(addr,
1093                                  atomics.getAtomicAlignment().getQuantity());
1094   }
1095
1096   // Do the atomic store.
1097   llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
1098   llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1099
1100   // Initializations don't need to be atomic.
1101   if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
1102
1103   // Other decoration.
1104   store->setAlignment(dest.getAlignment().getQuantity());
1105   if (dest.isVolatileQualified())
1106     store->setVolatile(true);
1107   if (dest.getTBAAInfo())
1108     CGM.DecorateInstruction(store, dest.getTBAAInfo());
1109 }
1110
1111 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1112   AtomicInfo atomics(*this, dest);
1113
1114   switch (atomics.getEvaluationKind()) {
1115   case TEK_Scalar: {
1116     llvm::Value *value = EmitScalarExpr(init);
1117     atomics.emitCopyIntoMemory(RValue::get(value), dest);
1118     return;
1119   }
1120
1121   case TEK_Complex: {
1122     ComplexPairTy value = EmitComplexExpr(init);
1123     atomics.emitCopyIntoMemory(RValue::getComplex(value), dest);
1124     return;
1125   }
1126
1127   case TEK_Aggregate: {
1128     // Fix up the destination if the initializer isn't an expression
1129     // of atomic type.
1130     bool Zeroed = false;
1131     if (!init->getType()->isAtomicType()) {
1132       Zeroed = atomics.emitMemSetZeroIfNecessary(dest);
1133       dest = atomics.projectValue(dest);
1134     }
1135
1136     // Evaluate the expression directly into the destination.
1137     AggValueSlot slot = AggValueSlot::forLValue(dest,
1138                                         AggValueSlot::IsNotDestructed,
1139                                         AggValueSlot::DoesNotNeedGCBarriers,
1140                                         AggValueSlot::IsNotAliased,
1141                                         Zeroed ? AggValueSlot::IsZeroed :
1142                                                  AggValueSlot::IsNotZeroed);
1143
1144     EmitAggExpr(init, slot);
1145     return;
1146   }
1147   }
1148   llvm_unreachable("bad evaluation kind");
1149 }