]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp
Merge llvm, clang, compiler-rt, libc++, libunwind, lld, lldb and openmp
[FreeBSD/FreeBSD.git] / contrib / llvm / tools / clang / lib / CodeGen / CGAtomic.cpp
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "CGCall.h"
15 #include "CGRecordLayout.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/ASTContext.h"
20 #include "clang/CodeGen/CGFunctionInfo.h"
21 #include "clang/Frontend/FrontendDiagnostic.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/Operator.h"
26
27 using namespace clang;
28 using namespace CodeGen;
29
30 namespace {
31   class AtomicInfo {
32     CodeGenFunction &CGF;
33     QualType AtomicTy;
34     QualType ValueTy;
35     uint64_t AtomicSizeInBits;
36     uint64_t ValueSizeInBits;
37     CharUnits AtomicAlign;
38     CharUnits ValueAlign;
39     CharUnits LValueAlign;
40     TypeEvaluationKind EvaluationKind;
41     bool UseLibcall;
42     LValue LVal;
43     CGBitFieldInfo BFI;
44   public:
45     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
46         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
47           EvaluationKind(TEK_Scalar), UseLibcall(true) {
48       assert(!lvalue.isGlobalReg());
49       ASTContext &C = CGF.getContext();
50       if (lvalue.isSimple()) {
51         AtomicTy = lvalue.getType();
52         if (auto *ATy = AtomicTy->getAs<AtomicType>())
53           ValueTy = ATy->getValueType();
54         else
55           ValueTy = AtomicTy;
56         EvaluationKind = CGF.getEvaluationKind(ValueTy);
57
58         uint64_t ValueAlignInBits;
59         uint64_t AtomicAlignInBits;
60         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
61         ValueSizeInBits = ValueTI.Width;
62         ValueAlignInBits = ValueTI.Align;
63
64         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
65         AtomicSizeInBits = AtomicTI.Width;
66         AtomicAlignInBits = AtomicTI.Align;
67
68         assert(ValueSizeInBits <= AtomicSizeInBits);
69         assert(ValueAlignInBits <= AtomicAlignInBits);
70
71         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
72         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
73         if (lvalue.getAlignment().isZero())
74           lvalue.setAlignment(AtomicAlign);
75
76         LVal = lvalue;
77       } else if (lvalue.isBitField()) {
78         ValueTy = lvalue.getType();
79         ValueSizeInBits = C.getTypeSize(ValueTy);
80         auto &OrigBFI = lvalue.getBitFieldInfo();
81         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
82         AtomicSizeInBits = C.toBits(
83             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
84                 .alignTo(lvalue.getAlignment()));
85         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
86         auto OffsetInChars =
87             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
88             lvalue.getAlignment();
89         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
90             VoidPtrAddr, OffsetInChars.getQuantity());
91         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
92             VoidPtrAddr,
93             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
94             "atomic_bitfield_base");
95         BFI = OrigBFI;
96         BFI.Offset = Offset;
97         BFI.StorageSize = AtomicSizeInBits;
98         BFI.StorageOffset += OffsetInChars;
99         LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
100                                     BFI, lvalue.getType(), lvalue.getBaseInfo(),
101                                     lvalue.getTBAAInfo());
102         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
103         if (AtomicTy.isNull()) {
104           llvm::APInt Size(
105               /*numBits=*/32,
106               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
107           AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
108                                             /*IndexTypeQuals=*/0);
109         }
110         AtomicAlign = ValueAlign = lvalue.getAlignment();
111       } else if (lvalue.isVectorElt()) {
112         ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
113         ValueSizeInBits = C.getTypeSize(ValueTy);
114         AtomicTy = lvalue.getType();
115         AtomicSizeInBits = C.getTypeSize(AtomicTy);
116         AtomicAlign = ValueAlign = lvalue.getAlignment();
117         LVal = lvalue;
118       } else {
119         assert(lvalue.isExtVectorElt());
120         ValueTy = lvalue.getType();
121         ValueSizeInBits = C.getTypeSize(ValueTy);
122         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
123             lvalue.getType(), lvalue.getExtVectorAddress()
124                                   .getElementType()->getVectorNumElements());
125         AtomicSizeInBits = C.getTypeSize(AtomicTy);
126         AtomicAlign = ValueAlign = lvalue.getAlignment();
127         LVal = lvalue;
128       }
129       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131     }
132
133     QualType getAtomicType() const { return AtomicTy; }
134     QualType getValueType() const { return ValueTy; }
135     CharUnits getAtomicAlignment() const { return AtomicAlign; }
136     CharUnits getValueAlignment() const { return ValueAlign; }
137     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
138     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
139     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
140     bool shouldUseLibcall() const { return UseLibcall; }
141     const LValue &getAtomicLValue() const { return LVal; }
142     llvm::Value *getAtomicPointer() const {
143       if (LVal.isSimple())
144         return LVal.getPointer();
145       else if (LVal.isBitField())
146         return LVal.getBitFieldPointer();
147       else if (LVal.isVectorElt())
148         return LVal.getVectorPointer();
149       assert(LVal.isExtVectorElt());
150       return LVal.getExtVectorPointer();
151     }
152     Address getAtomicAddress() const {
153       return Address(getAtomicPointer(), getAtomicAlignment());
154     }
155
156     Address getAtomicAddressAsAtomicIntPointer() const {
157       return emitCastToAtomicIntPointer(getAtomicAddress());
158     }
159
160     /// Is the atomic size larger than the underlying value type?
161     ///
162     /// Note that the absence of padding does not mean that atomic
163     /// objects are completely interchangeable with non-atomic
164     /// objects: we might have promoted the alignment of a type
165     /// without making it bigger.
166     bool hasPadding() const {
167       return (ValueSizeInBits != AtomicSizeInBits);
168     }
169
170     bool emitMemSetZeroIfNecessary() const;
171
172     llvm::Value *getAtomicSizeValue() const {
173       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
174       return CGF.CGM.getSize(size);
175     }
176
177     /// Cast the given pointer to an integer pointer suitable for atomic
178     /// operations if the source.
179     Address emitCastToAtomicIntPointer(Address Addr) const;
180
181     /// If Addr is compatible with the iN that will be used for an atomic
182     /// operation, bitcast it. Otherwise, create a temporary that is suitable
183     /// and copy the value across.
184     Address convertToAtomicIntPointer(Address Addr) const;
185
186     /// Turn an atomic-layout object into an r-value.
187     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
188                                      SourceLocation loc, bool AsValue) const;
189
190     /// Converts a rvalue to integer value.
191     llvm::Value *convertRValueToInt(RValue RVal) const;
192
193     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
194                                      AggValueSlot ResultSlot,
195                                      SourceLocation Loc, bool AsValue) const;
196
197     /// Copy an atomic r-value into atomic-layout memory.
198     void emitCopyIntoMemory(RValue rvalue) const;
199
200     /// Project an l-value down to the value field.
201     LValue projectValue() const {
202       assert(LVal.isSimple());
203       Address addr = getAtomicAddress();
204       if (hasPadding())
205         addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
206
207       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
208                               LVal.getBaseInfo(), LVal.getTBAAInfo());
209     }
210
211     /// Emits atomic load.
212     /// \returns Loaded value.
213     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
214                           bool AsValue, llvm::AtomicOrdering AO,
215                           bool IsVolatile);
216
217     /// Emits atomic compare-and-exchange sequence.
218     /// \param Expected Expected value.
219     /// \param Desired Desired value.
220     /// \param Success Atomic ordering for success operation.
221     /// \param Failure Atomic ordering for failed operation.
222     /// \param IsWeak true if atomic operation is weak, false otherwise.
223     /// \returns Pair of values: previous value from storage (value type) and
224     /// boolean flag (i1 type) with true if success and false otherwise.
225     std::pair<RValue, llvm::Value *>
226     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
227                               llvm::AtomicOrdering Success =
228                                   llvm::AtomicOrdering::SequentiallyConsistent,
229                               llvm::AtomicOrdering Failure =
230                                   llvm::AtomicOrdering::SequentiallyConsistent,
231                               bool IsWeak = false);
232
233     /// Emits atomic update.
234     /// \param AO Atomic ordering.
235     /// \param UpdateOp Update operation for the current lvalue.
236     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
237                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
238                           bool IsVolatile);
239     /// Emits atomic update.
240     /// \param AO Atomic ordering.
241     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
242                           bool IsVolatile);
243
244     /// Materialize an atomic r-value in atomic-layout memory.
245     Address materializeRValue(RValue rvalue) const;
246
247     /// Creates temp alloca for intermediate operations on atomic value.
248     Address CreateTempAlloca() const;
249   private:
250     bool requiresMemSetZero(llvm::Type *type) const;
251
252
253     /// Emits atomic load as a libcall.
254     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
255                                llvm::AtomicOrdering AO, bool IsVolatile);
256     /// Emits atomic load as LLVM instruction.
257     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
258     /// Emits atomic compare-and-exchange op as a libcall.
259     llvm::Value *EmitAtomicCompareExchangeLibcall(
260         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
261         llvm::AtomicOrdering Success =
262             llvm::AtomicOrdering::SequentiallyConsistent,
263         llvm::AtomicOrdering Failure =
264             llvm::AtomicOrdering::SequentiallyConsistent);
265     /// Emits atomic compare-and-exchange op as LLVM instruction.
266     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
267         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
268         llvm::AtomicOrdering Success =
269             llvm::AtomicOrdering::SequentiallyConsistent,
270         llvm::AtomicOrdering Failure =
271             llvm::AtomicOrdering::SequentiallyConsistent,
272         bool IsWeak = false);
273     /// Emit atomic update as libcalls.
274     void
275     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
276                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
277                             bool IsVolatile);
278     /// Emit atomic update as LLVM instructions.
279     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
280                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
281                             bool IsVolatile);
282     /// Emit atomic update as libcalls.
283     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
284                                  bool IsVolatile);
285     /// Emit atomic update as LLVM instructions.
286     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
287                             bool IsVolatile);
288   };
289 }
290
291 Address AtomicInfo::CreateTempAlloca() const {
292   Address TempAlloca = CGF.CreateMemTemp(
293       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
294                                                                 : AtomicTy,
295       getAtomicAlignment(),
296       "atomic-temp");
297   // Cast to pointer to value type for bitfields.
298   if (LVal.isBitField())
299     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
300         TempAlloca, getAtomicAddress().getType());
301   return TempAlloca;
302 }
303
304 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
305                                 StringRef fnName,
306                                 QualType resultType,
307                                 CallArgList &args) {
308   const CGFunctionInfo &fnInfo =
309     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
310   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
311   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
312   auto callee = CGCallee::forDirect(fn);
313   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
314 }
315
316 /// Does a store of the given IR type modify the full expected width?
317 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
318                            uint64_t expectedSize) {
319   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
320 }
321
322 /// Does the atomic type require memsetting to zero before initialization?
323 ///
324 /// The IR type is provided as a way of making certain queries faster.
325 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
326   // If the atomic type has size padding, we definitely need a memset.
327   if (hasPadding()) return true;
328
329   // Otherwise, do some simple heuristics to try to avoid it:
330   switch (getEvaluationKind()) {
331   // For scalars and complexes, check whether the store size of the
332   // type uses the full size.
333   case TEK_Scalar:
334     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
335   case TEK_Complex:
336     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
337                            AtomicSizeInBits / 2);
338
339   // Padding in structs has an undefined bit pattern.  User beware.
340   case TEK_Aggregate:
341     return false;
342   }
343   llvm_unreachable("bad evaluation kind");
344 }
345
346 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
347   assert(LVal.isSimple());
348   llvm::Value *addr = LVal.getPointer();
349   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
350     return false;
351
352   CGF.Builder.CreateMemSet(
353       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
354       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
355       LVal.getAlignment().getQuantity());
356   return true;
357 }
358
359 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
360                               Address Dest, Address Ptr,
361                               Address Val1, Address Val2,
362                               uint64_t Size,
363                               llvm::AtomicOrdering SuccessOrder,
364                               llvm::AtomicOrdering FailureOrder,
365                               llvm::SyncScope::ID Scope) {
366   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
367   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
368   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
369
370   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
371       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
372       Scope);
373   Pair->setVolatile(E->isVolatile());
374   Pair->setWeak(IsWeak);
375
376   // Cmp holds the result of the compare-exchange operation: true on success,
377   // false on failure.
378   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
379   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
380
381   // This basic block is used to hold the store instruction if the operation
382   // failed.
383   llvm::BasicBlock *StoreExpectedBB =
384       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
385
386   // This basic block is the exit point of the operation, we should end up
387   // here regardless of whether or not the operation succeeded.
388   llvm::BasicBlock *ContinueBB =
389       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
390
391   // Update Expected if Expected isn't equal to Old, otherwise branch to the
392   // exit point.
393   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
394
395   CGF.Builder.SetInsertPoint(StoreExpectedBB);
396   // Update the memory at Expected with Old's value.
397   CGF.Builder.CreateStore(Old, Val1);
398   // Finally, branch to the exit point.
399   CGF.Builder.CreateBr(ContinueBB);
400
401   CGF.Builder.SetInsertPoint(ContinueBB);
402   // Update the memory at Dest with Cmp's value.
403   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
404 }
405
406 /// Given an ordering required on success, emit all possible cmpxchg
407 /// instructions to cope with the provided (but possibly only dynamically known)
408 /// FailureOrder.
409 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
410                                         bool IsWeak, Address Dest, Address Ptr,
411                                         Address Val1, Address Val2,
412                                         llvm::Value *FailureOrderVal,
413                                         uint64_t Size,
414                                         llvm::AtomicOrdering SuccessOrder,
415                                         llvm::SyncScope::ID Scope) {
416   llvm::AtomicOrdering FailureOrder;
417   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
418     auto FOS = FO->getSExtValue();
419     if (!llvm::isValidAtomicOrderingCABI(FOS))
420       FailureOrder = llvm::AtomicOrdering::Monotonic;
421     else
422       switch ((llvm::AtomicOrderingCABI)FOS) {
423       case llvm::AtomicOrderingCABI::relaxed:
424       case llvm::AtomicOrderingCABI::release:
425       case llvm::AtomicOrderingCABI::acq_rel:
426         FailureOrder = llvm::AtomicOrdering::Monotonic;
427         break;
428       case llvm::AtomicOrderingCABI::consume:
429       case llvm::AtomicOrderingCABI::acquire:
430         FailureOrder = llvm::AtomicOrdering::Acquire;
431         break;
432       case llvm::AtomicOrderingCABI::seq_cst:
433         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
434         break;
435       }
436     if (isStrongerThan(FailureOrder, SuccessOrder)) {
437       // Don't assert on undefined behavior "failure argument shall be no
438       // stronger than the success argument".
439       FailureOrder =
440           llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
441     }
442     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
443                       FailureOrder, Scope);
444     return;
445   }
446
447   // Create all the relevant BB's
448   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
449                    *SeqCstBB = nullptr;
450   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
451   if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
452       SuccessOrder != llvm::AtomicOrdering::Release)
453     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
454   if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
455     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
456
457   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
458
459   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
460
461   // Emit all the different atomics
462
463   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
464   // doesn't matter unless someone is crazy enough to use something that
465   // doesn't fold to a constant for the ordering.
466   CGF.Builder.SetInsertPoint(MonotonicBB);
467   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
468                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
469   CGF.Builder.CreateBr(ContBB);
470
471   if (AcquireBB) {
472     CGF.Builder.SetInsertPoint(AcquireBB);
473     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
474                       Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
475     CGF.Builder.CreateBr(ContBB);
476     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
477                 AcquireBB);
478     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
479                 AcquireBB);
480   }
481   if (SeqCstBB) {
482     CGF.Builder.SetInsertPoint(SeqCstBB);
483     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
484                       llvm::AtomicOrdering::SequentiallyConsistent, Scope);
485     CGF.Builder.CreateBr(ContBB);
486     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
487                 SeqCstBB);
488   }
489
490   CGF.Builder.SetInsertPoint(ContBB);
491 }
492
493 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
494                          Address Ptr, Address Val1, Address Val2,
495                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
496                          uint64_t Size, llvm::AtomicOrdering Order,
497                          llvm::SyncScope::ID Scope) {
498   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
499   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
500
501   switch (E->getOp()) {
502   case AtomicExpr::AO__c11_atomic_init:
503   case AtomicExpr::AO__opencl_atomic_init:
504     llvm_unreachable("Already handled!");
505
506   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
507   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
508     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
509                                 FailureOrder, Size, Order, Scope);
510     return;
511   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
512   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
513     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
514                                 FailureOrder, Size, Order, Scope);
515     return;
516   case AtomicExpr::AO__atomic_compare_exchange:
517   case AtomicExpr::AO__atomic_compare_exchange_n: {
518     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
519       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
520                                   Val1, Val2, FailureOrder, Size, Order, Scope);
521     } else {
522       // Create all the relevant BB's
523       llvm::BasicBlock *StrongBB =
524           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
525       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
526       llvm::BasicBlock *ContBB =
527           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
528
529       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
530       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
531
532       CGF.Builder.SetInsertPoint(StrongBB);
533       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
534                                   FailureOrder, Size, Order, Scope);
535       CGF.Builder.CreateBr(ContBB);
536
537       CGF.Builder.SetInsertPoint(WeakBB);
538       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
539                                   FailureOrder, Size, Order, Scope);
540       CGF.Builder.CreateBr(ContBB);
541
542       CGF.Builder.SetInsertPoint(ContBB);
543     }
544     return;
545   }
546   case AtomicExpr::AO__c11_atomic_load:
547   case AtomicExpr::AO__opencl_atomic_load:
548   case AtomicExpr::AO__atomic_load_n:
549   case AtomicExpr::AO__atomic_load: {
550     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
551     Load->setAtomic(Order, Scope);
552     Load->setVolatile(E->isVolatile());
553     CGF.Builder.CreateStore(Load, Dest);
554     return;
555   }
556
557   case AtomicExpr::AO__c11_atomic_store:
558   case AtomicExpr::AO__opencl_atomic_store:
559   case AtomicExpr::AO__atomic_store:
560   case AtomicExpr::AO__atomic_store_n: {
561     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
562     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
563     Store->setAtomic(Order, Scope);
564     Store->setVolatile(E->isVolatile());
565     return;
566   }
567
568   case AtomicExpr::AO__c11_atomic_exchange:
569   case AtomicExpr::AO__opencl_atomic_exchange:
570   case AtomicExpr::AO__atomic_exchange_n:
571   case AtomicExpr::AO__atomic_exchange:
572     Op = llvm::AtomicRMWInst::Xchg;
573     break;
574
575   case AtomicExpr::AO__atomic_add_fetch:
576     PostOp = llvm::Instruction::Add;
577     LLVM_FALLTHROUGH;
578   case AtomicExpr::AO__c11_atomic_fetch_add:
579   case AtomicExpr::AO__opencl_atomic_fetch_add:
580   case AtomicExpr::AO__atomic_fetch_add:
581     Op = llvm::AtomicRMWInst::Add;
582     break;
583
584   case AtomicExpr::AO__atomic_sub_fetch:
585     PostOp = llvm::Instruction::Sub;
586     LLVM_FALLTHROUGH;
587   case AtomicExpr::AO__c11_atomic_fetch_sub:
588   case AtomicExpr::AO__opencl_atomic_fetch_sub:
589   case AtomicExpr::AO__atomic_fetch_sub:
590     Op = llvm::AtomicRMWInst::Sub;
591     break;
592
593   case AtomicExpr::AO__opencl_atomic_fetch_min:
594   case AtomicExpr::AO__atomic_fetch_min:
595     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
596                                                   : llvm::AtomicRMWInst::UMin;
597     break;
598
599   case AtomicExpr::AO__opencl_atomic_fetch_max:
600   case AtomicExpr::AO__atomic_fetch_max:
601     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
602                                                   : llvm::AtomicRMWInst::UMax;
603     break;
604
605   case AtomicExpr::AO__atomic_and_fetch:
606     PostOp = llvm::Instruction::And;
607     LLVM_FALLTHROUGH;
608   case AtomicExpr::AO__c11_atomic_fetch_and:
609   case AtomicExpr::AO__opencl_atomic_fetch_and:
610   case AtomicExpr::AO__atomic_fetch_and:
611     Op = llvm::AtomicRMWInst::And;
612     break;
613
614   case AtomicExpr::AO__atomic_or_fetch:
615     PostOp = llvm::Instruction::Or;
616     LLVM_FALLTHROUGH;
617   case AtomicExpr::AO__c11_atomic_fetch_or:
618   case AtomicExpr::AO__opencl_atomic_fetch_or:
619   case AtomicExpr::AO__atomic_fetch_or:
620     Op = llvm::AtomicRMWInst::Or;
621     break;
622
623   case AtomicExpr::AO__atomic_xor_fetch:
624     PostOp = llvm::Instruction::Xor;
625     LLVM_FALLTHROUGH;
626   case AtomicExpr::AO__c11_atomic_fetch_xor:
627   case AtomicExpr::AO__opencl_atomic_fetch_xor:
628   case AtomicExpr::AO__atomic_fetch_xor:
629     Op = llvm::AtomicRMWInst::Xor;
630     break;
631
632   case AtomicExpr::AO__atomic_nand_fetch:
633     PostOp = llvm::Instruction::And; // the NOT is special cased below
634     LLVM_FALLTHROUGH;
635   case AtomicExpr::AO__atomic_fetch_nand:
636     Op = llvm::AtomicRMWInst::Nand;
637     break;
638   }
639
640   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
641   llvm::AtomicRMWInst *RMWI =
642       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
643   RMWI->setVolatile(E->isVolatile());
644
645   // For __atomic_*_fetch operations, perform the operation again to
646   // determine the value which was written.
647   llvm::Value *Result = RMWI;
648   if (PostOp)
649     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
650   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
651     Result = CGF.Builder.CreateNot(Result);
652   CGF.Builder.CreateStore(Result, Dest);
653 }
654
655 // This function emits any expression (scalar, complex, or aggregate)
656 // into a temporary alloca.
657 static Address
658 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
659   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
660   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
661                        /*Init*/ true);
662   return DeclPtr;
663 }
664
665 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
666                          Address Ptr, Address Val1, Address Val2,
667                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
668                          uint64_t Size, llvm::AtomicOrdering Order,
669                          llvm::Value *Scope) {
670   auto ScopeModel = Expr->getScopeModel();
671
672   // LLVM atomic instructions always have synch scope. If clang atomic
673   // expression has no scope operand, use default LLVM synch scope.
674   if (!ScopeModel) {
675     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
676                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
677     return;
678   }
679
680   // Handle constant scope.
681   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
682     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
683         ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext());
684     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
685                  Order, SCID);
686     return;
687   }
688
689   // Handle non-constant scope.
690   auto &Builder = CGF.Builder;
691   auto Scopes = ScopeModel->getRuntimeValues();
692   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
693   for (auto S : Scopes)
694     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
695
696   llvm::BasicBlock *ContBB =
697       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
698
699   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
700   // If unsupported synch scope is encountered at run time, assume a fallback
701   // synch scope value.
702   auto FallBack = ScopeModel->getFallBackValue();
703   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
704   for (auto S : Scopes) {
705     auto *B = BB[S];
706     if (S != FallBack)
707       SI->addCase(Builder.getInt32(S), B);
708
709     Builder.SetInsertPoint(B);
710     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
711                  Order,
712                  CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S),
713                                                          CGF.getLLVMContext()));
714     Builder.CreateBr(ContBB);
715   }
716
717   Builder.SetInsertPoint(ContBB);
718 }
719
720 static void
721 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
722                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
723                   SourceLocation Loc, CharUnits SizeInChars) {
724   if (UseOptimizedLibcall) {
725     // Load value and pass it to the function directly.
726     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
727     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
728     ValTy =
729         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
730     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
731                                                 SizeInBits)->getPointerTo();
732     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
733     Val = CGF.EmitLoadOfScalar(Ptr, false,
734                                CGF.getContext().getPointerType(ValTy),
735                                Loc);
736     // Coerce the value into an appropriately sized integer type.
737     Args.add(RValue::get(Val), ValTy);
738   } else {
739     // Non-optimized functions always take a reference.
740     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
741                          CGF.getContext().VoidPtrTy);
742   }
743 }
744
745 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
746   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
747   QualType MemTy = AtomicTy;
748   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
749     MemTy = AT->getValueType();
750   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
751
752   Address Val1 = Address::invalid();
753   Address Val2 = Address::invalid();
754   Address Dest = Address::invalid();
755   Address Ptr = EmitPointerWithAlignment(E->getPtr());
756
757   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
758       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
759     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
760     EmitAtomicInit(E->getVal1(), lvalue);
761     return RValue::get(nullptr);
762   }
763
764   CharUnits sizeChars, alignChars;
765   std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
766   uint64_t Size = sizeChars.getQuantity();
767   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
768
769   bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits;
770   bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0;
771   bool UseLibcall = Misaligned | Oversized;
772
773   if (UseLibcall) {
774     CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
775         << !Oversized;
776   }
777
778   llvm::Value *Order = EmitScalarExpr(E->getOrder());
779   llvm::Value *Scope =
780       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
781
782   switch (E->getOp()) {
783   case AtomicExpr::AO__c11_atomic_init:
784   case AtomicExpr::AO__opencl_atomic_init:
785     llvm_unreachable("Already handled above with EmitAtomicInit!");
786
787   case AtomicExpr::AO__c11_atomic_load:
788   case AtomicExpr::AO__opencl_atomic_load:
789   case AtomicExpr::AO__atomic_load_n:
790     break;
791
792   case AtomicExpr::AO__atomic_load:
793     Dest = EmitPointerWithAlignment(E->getVal1());
794     break;
795
796   case AtomicExpr::AO__atomic_store:
797     Val1 = EmitPointerWithAlignment(E->getVal1());
798     break;
799
800   case AtomicExpr::AO__atomic_exchange:
801     Val1 = EmitPointerWithAlignment(E->getVal1());
802     Dest = EmitPointerWithAlignment(E->getVal2());
803     break;
804
805   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
806   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
807   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
808   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
809   case AtomicExpr::AO__atomic_compare_exchange_n:
810   case AtomicExpr::AO__atomic_compare_exchange:
811     Val1 = EmitPointerWithAlignment(E->getVal1());
812     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
813       Val2 = EmitPointerWithAlignment(E->getVal2());
814     else
815       Val2 = EmitValToTemp(*this, E->getVal2());
816     OrderFail = EmitScalarExpr(E->getOrderFail());
817     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
818         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
819       IsWeak = EmitScalarExpr(E->getWeak());
820     break;
821
822   case AtomicExpr::AO__c11_atomic_fetch_add:
823   case AtomicExpr::AO__c11_atomic_fetch_sub:
824   case AtomicExpr::AO__opencl_atomic_fetch_add:
825   case AtomicExpr::AO__opencl_atomic_fetch_sub:
826     if (MemTy->isPointerType()) {
827       // For pointer arithmetic, we're required to do a bit of math:
828       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
829       // ... but only for the C11 builtins. The GNU builtins expect the
830       // user to multiply by sizeof(T).
831       QualType Val1Ty = E->getVal1()->getType();
832       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
833       CharUnits PointeeIncAmt =
834           getContext().getTypeSizeInChars(MemTy->getPointeeType());
835       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
836       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
837       Val1 = Temp;
838       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
839       break;
840     }
841       LLVM_FALLTHROUGH;
842   case AtomicExpr::AO__atomic_fetch_add:
843   case AtomicExpr::AO__atomic_fetch_sub:
844   case AtomicExpr::AO__atomic_add_fetch:
845   case AtomicExpr::AO__atomic_sub_fetch:
846   case AtomicExpr::AO__c11_atomic_store:
847   case AtomicExpr::AO__c11_atomic_exchange:
848   case AtomicExpr::AO__opencl_atomic_store:
849   case AtomicExpr::AO__opencl_atomic_exchange:
850   case AtomicExpr::AO__atomic_store_n:
851   case AtomicExpr::AO__atomic_exchange_n:
852   case AtomicExpr::AO__c11_atomic_fetch_and:
853   case AtomicExpr::AO__c11_atomic_fetch_or:
854   case AtomicExpr::AO__c11_atomic_fetch_xor:
855   case AtomicExpr::AO__opencl_atomic_fetch_and:
856   case AtomicExpr::AO__opencl_atomic_fetch_or:
857   case AtomicExpr::AO__opencl_atomic_fetch_xor:
858   case AtomicExpr::AO__opencl_atomic_fetch_min:
859   case AtomicExpr::AO__opencl_atomic_fetch_max:
860   case AtomicExpr::AO__atomic_fetch_and:
861   case AtomicExpr::AO__atomic_fetch_or:
862   case AtomicExpr::AO__atomic_fetch_xor:
863   case AtomicExpr::AO__atomic_fetch_nand:
864   case AtomicExpr::AO__atomic_and_fetch:
865   case AtomicExpr::AO__atomic_or_fetch:
866   case AtomicExpr::AO__atomic_xor_fetch:
867   case AtomicExpr::AO__atomic_nand_fetch:
868   case AtomicExpr::AO__atomic_fetch_min:
869   case AtomicExpr::AO__atomic_fetch_max:
870     Val1 = EmitValToTemp(*this, E->getVal1());
871     break;
872   }
873
874   QualType RValTy = E->getType().getUnqualifiedType();
875
876   // The inlined atomics only function on iN types, where N is a power of 2. We
877   // need to make sure (via temporaries if necessary) that all incoming values
878   // are compatible.
879   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
880   AtomicInfo Atomics(*this, AtomicVal);
881
882   Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
883   if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
884   if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
885   if (Dest.isValid())
886     Dest = Atomics.emitCastToAtomicIntPointer(Dest);
887   else if (E->isCmpXChg())
888     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
889   else if (!RValTy->isVoidType())
890     Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
891
892   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
893   if (UseLibcall) {
894     bool UseOptimizedLibcall = false;
895     switch (E->getOp()) {
896     case AtomicExpr::AO__c11_atomic_init:
897     case AtomicExpr::AO__opencl_atomic_init:
898       llvm_unreachable("Already handled above with EmitAtomicInit!");
899
900     case AtomicExpr::AO__c11_atomic_fetch_add:
901     case AtomicExpr::AO__opencl_atomic_fetch_add:
902     case AtomicExpr::AO__atomic_fetch_add:
903     case AtomicExpr::AO__c11_atomic_fetch_and:
904     case AtomicExpr::AO__opencl_atomic_fetch_and:
905     case AtomicExpr::AO__atomic_fetch_and:
906     case AtomicExpr::AO__c11_atomic_fetch_or:
907     case AtomicExpr::AO__opencl_atomic_fetch_or:
908     case AtomicExpr::AO__atomic_fetch_or:
909     case AtomicExpr::AO__atomic_fetch_nand:
910     case AtomicExpr::AO__c11_atomic_fetch_sub:
911     case AtomicExpr::AO__opencl_atomic_fetch_sub:
912     case AtomicExpr::AO__atomic_fetch_sub:
913     case AtomicExpr::AO__c11_atomic_fetch_xor:
914     case AtomicExpr::AO__opencl_atomic_fetch_xor:
915     case AtomicExpr::AO__opencl_atomic_fetch_min:
916     case AtomicExpr::AO__opencl_atomic_fetch_max:
917     case AtomicExpr::AO__atomic_fetch_xor:
918     case AtomicExpr::AO__atomic_add_fetch:
919     case AtomicExpr::AO__atomic_and_fetch:
920     case AtomicExpr::AO__atomic_nand_fetch:
921     case AtomicExpr::AO__atomic_or_fetch:
922     case AtomicExpr::AO__atomic_sub_fetch:
923     case AtomicExpr::AO__atomic_xor_fetch:
924     case AtomicExpr::AO__atomic_fetch_min:
925     case AtomicExpr::AO__atomic_fetch_max:
926       // For these, only library calls for certain sizes exist.
927       UseOptimizedLibcall = true;
928       break;
929
930     case AtomicExpr::AO__atomic_load:
931     case AtomicExpr::AO__atomic_store:
932     case AtomicExpr::AO__atomic_exchange:
933     case AtomicExpr::AO__atomic_compare_exchange:
934       // Use the generic version if we don't know that the operand will be
935       // suitably aligned for the optimized version.
936       if (Misaligned)
937         break;
938       LLVM_FALLTHROUGH;
939     case AtomicExpr::AO__c11_atomic_load:
940     case AtomicExpr::AO__c11_atomic_store:
941     case AtomicExpr::AO__c11_atomic_exchange:
942     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
943     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
944     case AtomicExpr::AO__opencl_atomic_load:
945     case AtomicExpr::AO__opencl_atomic_store:
946     case AtomicExpr::AO__opencl_atomic_exchange:
947     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
948     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
949     case AtomicExpr::AO__atomic_load_n:
950     case AtomicExpr::AO__atomic_store_n:
951     case AtomicExpr::AO__atomic_exchange_n:
952     case AtomicExpr::AO__atomic_compare_exchange_n:
953       // Only use optimized library calls for sizes for which they exist.
954       // FIXME: Size == 16 optimized library functions exist too.
955       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
956         UseOptimizedLibcall = true;
957       break;
958     }
959
960     CallArgList Args;
961     if (!UseOptimizedLibcall) {
962       // For non-optimized library calls, the size is the first parameter
963       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
964                getContext().getSizeType());
965     }
966     // Atomic address is the first or second parameter
967     // The OpenCL atomic library functions only accept pointer arguments to
968     // generic address space.
969     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
970       if (!E->isOpenCL())
971         return V;
972       auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
973       if (AS == LangAS::opencl_generic)
974         return V;
975       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
976       auto T = V->getType();
977       auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
978
979       return getTargetHooks().performAddrSpaceCast(
980           *this, V, AS, LangAS::opencl_generic, DestType, false);
981     };
982
983     Args.add(RValue::get(CastToGenericAddrSpace(
984                  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
985              getContext().VoidPtrTy);
986
987     std::string LibCallName;
988     QualType LoweredMemTy =
989       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
990     QualType RetTy;
991     bool HaveRetTy = false;
992     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
993     switch (E->getOp()) {
994     case AtomicExpr::AO__c11_atomic_init:
995     case AtomicExpr::AO__opencl_atomic_init:
996       llvm_unreachable("Already handled!");
997
998     // There is only one libcall for compare an exchange, because there is no
999     // optimisation benefit possible from a libcall version of a weak compare
1000     // and exchange.
1001     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1002     //                                void *desired, int success, int failure)
1003     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1004     //                                  int success, int failure)
1005     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1006     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1007     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1008     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1009     case AtomicExpr::AO__atomic_compare_exchange:
1010     case AtomicExpr::AO__atomic_compare_exchange_n:
1011       LibCallName = "__atomic_compare_exchange";
1012       RetTy = getContext().BoolTy;
1013       HaveRetTy = true;
1014       Args.add(
1015           RValue::get(CastToGenericAddrSpace(
1016               EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1017           getContext().VoidPtrTy);
1018       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1019                         MemTy, E->getExprLoc(), sizeChars);
1020       Args.add(RValue::get(Order), getContext().IntTy);
1021       Order = OrderFail;
1022       break;
1023     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1024     //                        int order)
1025     // T __atomic_exchange_N(T *mem, T val, int order)
1026     case AtomicExpr::AO__c11_atomic_exchange:
1027     case AtomicExpr::AO__opencl_atomic_exchange:
1028     case AtomicExpr::AO__atomic_exchange_n:
1029     case AtomicExpr::AO__atomic_exchange:
1030       LibCallName = "__atomic_exchange";
1031       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1032                         MemTy, E->getExprLoc(), sizeChars);
1033       break;
1034     // void __atomic_store(size_t size, void *mem, void *val, int order)
1035     // void __atomic_store_N(T *mem, T val, int order)
1036     case AtomicExpr::AO__c11_atomic_store:
1037     case AtomicExpr::AO__opencl_atomic_store:
1038     case AtomicExpr::AO__atomic_store:
1039     case AtomicExpr::AO__atomic_store_n:
1040       LibCallName = "__atomic_store";
1041       RetTy = getContext().VoidTy;
1042       HaveRetTy = true;
1043       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1044                         MemTy, E->getExprLoc(), sizeChars);
1045       break;
1046     // void __atomic_load(size_t size, void *mem, void *return, int order)
1047     // T __atomic_load_N(T *mem, int order)
1048     case AtomicExpr::AO__c11_atomic_load:
1049     case AtomicExpr::AO__opencl_atomic_load:
1050     case AtomicExpr::AO__atomic_load:
1051     case AtomicExpr::AO__atomic_load_n:
1052       LibCallName = "__atomic_load";
1053       break;
1054     // T __atomic_add_fetch_N(T *mem, T val, int order)
1055     // T __atomic_fetch_add_N(T *mem, T val, int order)
1056     case AtomicExpr::AO__atomic_add_fetch:
1057       PostOp = llvm::Instruction::Add;
1058       LLVM_FALLTHROUGH;
1059     case AtomicExpr::AO__c11_atomic_fetch_add:
1060     case AtomicExpr::AO__opencl_atomic_fetch_add:
1061     case AtomicExpr::AO__atomic_fetch_add:
1062       LibCallName = "__atomic_fetch_add";
1063       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1064                         LoweredMemTy, E->getExprLoc(), sizeChars);
1065       break;
1066     // T __atomic_and_fetch_N(T *mem, T val, int order)
1067     // T __atomic_fetch_and_N(T *mem, T val, int order)
1068     case AtomicExpr::AO__atomic_and_fetch:
1069       PostOp = llvm::Instruction::And;
1070       LLVM_FALLTHROUGH;
1071     case AtomicExpr::AO__c11_atomic_fetch_and:
1072     case AtomicExpr::AO__opencl_atomic_fetch_and:
1073     case AtomicExpr::AO__atomic_fetch_and:
1074       LibCallName = "__atomic_fetch_and";
1075       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1076                         MemTy, E->getExprLoc(), sizeChars);
1077       break;
1078     // T __atomic_or_fetch_N(T *mem, T val, int order)
1079     // T __atomic_fetch_or_N(T *mem, T val, int order)
1080     case AtomicExpr::AO__atomic_or_fetch:
1081       PostOp = llvm::Instruction::Or;
1082       LLVM_FALLTHROUGH;
1083     case AtomicExpr::AO__c11_atomic_fetch_or:
1084     case AtomicExpr::AO__opencl_atomic_fetch_or:
1085     case AtomicExpr::AO__atomic_fetch_or:
1086       LibCallName = "__atomic_fetch_or";
1087       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1088                         MemTy, E->getExprLoc(), sizeChars);
1089       break;
1090     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1091     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1092     case AtomicExpr::AO__atomic_sub_fetch:
1093       PostOp = llvm::Instruction::Sub;
1094       LLVM_FALLTHROUGH;
1095     case AtomicExpr::AO__c11_atomic_fetch_sub:
1096     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1097     case AtomicExpr::AO__atomic_fetch_sub:
1098       LibCallName = "__atomic_fetch_sub";
1099       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1100                         LoweredMemTy, E->getExprLoc(), sizeChars);
1101       break;
1102     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1103     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1104     case AtomicExpr::AO__atomic_xor_fetch:
1105       PostOp = llvm::Instruction::Xor;
1106       LLVM_FALLTHROUGH;
1107     case AtomicExpr::AO__c11_atomic_fetch_xor:
1108     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1109     case AtomicExpr::AO__atomic_fetch_xor:
1110       LibCallName = "__atomic_fetch_xor";
1111       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1112                         MemTy, E->getExprLoc(), sizeChars);
1113       break;
1114     case AtomicExpr::AO__atomic_fetch_min:
1115     case AtomicExpr::AO__opencl_atomic_fetch_min:
1116       LibCallName = E->getValueType()->isSignedIntegerType()
1117                         ? "__atomic_fetch_min"
1118                         : "__atomic_fetch_umin";
1119       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1120                         LoweredMemTy, E->getExprLoc(), sizeChars);
1121       break;
1122     case AtomicExpr::AO__atomic_fetch_max:
1123     case AtomicExpr::AO__opencl_atomic_fetch_max:
1124       LibCallName = E->getValueType()->isSignedIntegerType()
1125                         ? "__atomic_fetch_max"
1126                         : "__atomic_fetch_umax";
1127       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1128                         LoweredMemTy, E->getExprLoc(), sizeChars);
1129       break;
1130     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1131     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1132     case AtomicExpr::AO__atomic_nand_fetch:
1133       PostOp = llvm::Instruction::And; // the NOT is special cased below
1134       LLVM_FALLTHROUGH;
1135     case AtomicExpr::AO__atomic_fetch_nand:
1136       LibCallName = "__atomic_fetch_nand";
1137       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1138                         MemTy, E->getExprLoc(), sizeChars);
1139       break;
1140     }
1141
1142     if (E->isOpenCL()) {
1143       LibCallName = std::string("__opencl") +
1144           StringRef(LibCallName).drop_front(1).str();
1145
1146     }
1147     // Optimized functions have the size in their name.
1148     if (UseOptimizedLibcall)
1149       LibCallName += "_" + llvm::utostr(Size);
1150     // By default, assume we return a value of the atomic type.
1151     if (!HaveRetTy) {
1152       if (UseOptimizedLibcall) {
1153         // Value is returned directly.
1154         // The function returns an appropriately sized integer type.
1155         RetTy = getContext().getIntTypeForBitwidth(
1156             getContext().toBits(sizeChars), /*Signed=*/false);
1157       } else {
1158         // Value is returned through parameter before the order.
1159         RetTy = getContext().VoidTy;
1160         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1161                  getContext().VoidPtrTy);
1162       }
1163     }
1164     // order is always the last parameter
1165     Args.add(RValue::get(Order),
1166              getContext().IntTy);
1167     if (E->isOpenCL())
1168       Args.add(RValue::get(Scope), getContext().IntTy);
1169
1170     // PostOp is only needed for the atomic_*_fetch operations, and
1171     // thus is only needed for and implemented in the
1172     // UseOptimizedLibcall codepath.
1173     assert(UseOptimizedLibcall || !PostOp);
1174
1175     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1176     // The value is returned directly from the libcall.
1177     if (E->isCmpXChg())
1178       return Res;
1179
1180     // The value is returned directly for optimized libcalls but the expr
1181     // provided an out-param.
1182     if (UseOptimizedLibcall && Res.getScalarVal()) {
1183       llvm::Value *ResVal = Res.getScalarVal();
1184       if (PostOp) {
1185         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1186         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1187       }
1188       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1189         ResVal = Builder.CreateNot(ResVal);
1190
1191       Builder.CreateStore(
1192           ResVal,
1193           Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1194     }
1195
1196     if (RValTy->isVoidType())
1197       return RValue::get(nullptr);
1198
1199     return convertTempToRValue(
1200         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1201         RValTy, E->getExprLoc());
1202   }
1203
1204   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1205                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1206                  E->getOp() == AtomicExpr::AO__atomic_store ||
1207                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1208   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1209                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1210                 E->getOp() == AtomicExpr::AO__atomic_load ||
1211                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1212
1213   if (isa<llvm::ConstantInt>(Order)) {
1214     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1215     // We should not ever get to a case where the ordering isn't a valid C ABI
1216     // value, but it's hard to enforce that in general.
1217     if (llvm::isValidAtomicOrderingCABI(ord))
1218       switch ((llvm::AtomicOrderingCABI)ord) {
1219       case llvm::AtomicOrderingCABI::relaxed:
1220         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1221                      llvm::AtomicOrdering::Monotonic, Scope);
1222         break;
1223       case llvm::AtomicOrderingCABI::consume:
1224       case llvm::AtomicOrderingCABI::acquire:
1225         if (IsStore)
1226           break; // Avoid crashing on code with undefined behavior
1227         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1228                      llvm::AtomicOrdering::Acquire, Scope);
1229         break;
1230       case llvm::AtomicOrderingCABI::release:
1231         if (IsLoad)
1232           break; // Avoid crashing on code with undefined behavior
1233         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1234                      llvm::AtomicOrdering::Release, Scope);
1235         break;
1236       case llvm::AtomicOrderingCABI::acq_rel:
1237         if (IsLoad || IsStore)
1238           break; // Avoid crashing on code with undefined behavior
1239         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1240                      llvm::AtomicOrdering::AcquireRelease, Scope);
1241         break;
1242       case llvm::AtomicOrderingCABI::seq_cst:
1243         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1244                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1245         break;
1246       }
1247     if (RValTy->isVoidType())
1248       return RValue::get(nullptr);
1249
1250     return convertTempToRValue(
1251         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1252                                         Dest.getAddressSpace())),
1253         RValTy, E->getExprLoc());
1254   }
1255
1256   // Long case, when Order isn't obviously constant.
1257
1258   // Create all the relevant BB's
1259   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1260                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1261                    *SeqCstBB = nullptr;
1262   MonotonicBB = createBasicBlock("monotonic", CurFn);
1263   if (!IsStore)
1264     AcquireBB = createBasicBlock("acquire", CurFn);
1265   if (!IsLoad)
1266     ReleaseBB = createBasicBlock("release", CurFn);
1267   if (!IsLoad && !IsStore)
1268     AcqRelBB = createBasicBlock("acqrel", CurFn);
1269   SeqCstBB = createBasicBlock("seqcst", CurFn);
1270   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1271
1272   // Create the switch for the split
1273   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1274   // doesn't matter unless someone is crazy enough to use something that
1275   // doesn't fold to a constant for the ordering.
1276   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1277   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1278
1279   // Emit all the different atomics
1280   Builder.SetInsertPoint(MonotonicBB);
1281   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1282                llvm::AtomicOrdering::Monotonic, Scope);
1283   Builder.CreateBr(ContBB);
1284   if (!IsStore) {
1285     Builder.SetInsertPoint(AcquireBB);
1286     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1287                  llvm::AtomicOrdering::Acquire, Scope);
1288     Builder.CreateBr(ContBB);
1289     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1290                 AcquireBB);
1291     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1292                 AcquireBB);
1293   }
1294   if (!IsLoad) {
1295     Builder.SetInsertPoint(ReleaseBB);
1296     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1297                  llvm::AtomicOrdering::Release, Scope);
1298     Builder.CreateBr(ContBB);
1299     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1300                 ReleaseBB);
1301   }
1302   if (!IsLoad && !IsStore) {
1303     Builder.SetInsertPoint(AcqRelBB);
1304     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305                  llvm::AtomicOrdering::AcquireRelease, Scope);
1306     Builder.CreateBr(ContBB);
1307     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1308                 AcqRelBB);
1309   }
1310   Builder.SetInsertPoint(SeqCstBB);
1311   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1312                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1313   Builder.CreateBr(ContBB);
1314   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1315               SeqCstBB);
1316
1317   // Cleanup and return
1318   Builder.SetInsertPoint(ContBB);
1319   if (RValTy->isVoidType())
1320     return RValue::get(nullptr);
1321
1322   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1323   return convertTempToRValue(
1324       Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1325                                       Dest.getAddressSpace())),
1326       RValTy, E->getExprLoc());
1327 }
1328
1329 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1330   unsigned addrspace =
1331     cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1332   llvm::IntegerType *ty =
1333     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1334   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1335 }
1336
1337 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1338   llvm::Type *Ty = Addr.getElementType();
1339   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1340   if (SourceSizeInBits != AtomicSizeInBits) {
1341     Address Tmp = CreateTempAlloca();
1342     CGF.Builder.CreateMemCpy(Tmp, Addr,
1343                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1344     Addr = Tmp;
1345   }
1346
1347   return emitCastToAtomicIntPointer(Addr);
1348 }
1349
1350 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1351                                              AggValueSlot resultSlot,
1352                                              SourceLocation loc,
1353                                              bool asValue) const {
1354   if (LVal.isSimple()) {
1355     if (EvaluationKind == TEK_Aggregate)
1356       return resultSlot.asRValue();
1357
1358     // Drill into the padding structure if we have one.
1359     if (hasPadding())
1360       addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1361
1362     // Otherwise, just convert the temporary to an r-value using the
1363     // normal conversion routine.
1364     return CGF.convertTempToRValue(addr, getValueType(), loc);
1365   }
1366   if (!asValue)
1367     // Get RValue from temp memory as atomic for non-simple lvalues
1368     return RValue::get(CGF.Builder.CreateLoad(addr));
1369   if (LVal.isBitField())
1370     return CGF.EmitLoadOfBitfieldLValue(
1371         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1372                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1373   if (LVal.isVectorElt())
1374     return CGF.EmitLoadOfLValue(
1375         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1376                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1377   assert(LVal.isExtVectorElt());
1378   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1379       addr, LVal.getExtVectorElts(), LVal.getType(),
1380       LVal.getBaseInfo(), TBAAAccessInfo()));
1381 }
1382
1383 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1384                                              AggValueSlot ResultSlot,
1385                                              SourceLocation Loc,
1386                                              bool AsValue) const {
1387   // Try not to in some easy cases.
1388   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1389   if (getEvaluationKind() == TEK_Scalar &&
1390       (((!LVal.isBitField() ||
1391          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1392         !hasPadding()) ||
1393        !AsValue)) {
1394     auto *ValTy = AsValue
1395                       ? CGF.ConvertTypeForMem(ValueTy)
1396                       : getAtomicAddress().getType()->getPointerElementType();
1397     if (ValTy->isIntegerTy()) {
1398       assert(IntVal->getType() == ValTy && "Different integer types.");
1399       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1400     } else if (ValTy->isPointerTy())
1401       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1402     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1403       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1404   }
1405
1406   // Create a temporary.  This needs to be big enough to hold the
1407   // atomic integer.
1408   Address Temp = Address::invalid();
1409   bool TempIsVolatile = false;
1410   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1411     assert(!ResultSlot.isIgnored());
1412     Temp = ResultSlot.getAddress();
1413     TempIsVolatile = ResultSlot.isVolatile();
1414   } else {
1415     Temp = CreateTempAlloca();
1416   }
1417
1418   // Slam the integer into the temporary.
1419   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1420   CGF.Builder.CreateStore(IntVal, CastTemp)
1421       ->setVolatile(TempIsVolatile);
1422
1423   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1424 }
1425
1426 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1427                                        llvm::AtomicOrdering AO, bool) {
1428   // void __atomic_load(size_t size, void *mem, void *return, int order);
1429   CallArgList Args;
1430   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1431   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1432            CGF.getContext().VoidPtrTy);
1433   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1434            CGF.getContext().VoidPtrTy);
1435   Args.add(
1436       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1437       CGF.getContext().IntTy);
1438   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1439 }
1440
1441 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1442                                           bool IsVolatile) {
1443   // Okay, we're doing this natively.
1444   Address Addr = getAtomicAddressAsAtomicIntPointer();
1445   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1446   Load->setAtomic(AO);
1447
1448   // Other decoration.
1449   if (IsVolatile)
1450     Load->setVolatile(true);
1451   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1452   return Load;
1453 }
1454
1455 /// An LValue is a candidate for having its loads and stores be made atomic if
1456 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1457 /// performing such an operation can be performed without a libcall.
1458 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1459   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1460   AtomicInfo AI(*this, LV);
1461   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1462   // An atomic is inline if we don't need to use a libcall.
1463   bool AtomicIsInline = !AI.shouldUseLibcall();
1464   // MSVC doesn't seem to do this for types wider than a pointer.
1465   if (getContext().getTypeSize(LV.getType()) >
1466       getContext().getTypeSize(getContext().getIntPtrType()))
1467     return false;
1468   return IsVolatile && AtomicIsInline;
1469 }
1470
1471 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1472                                        AggValueSlot Slot) {
1473   llvm::AtomicOrdering AO;
1474   bool IsVolatile = LV.isVolatileQualified();
1475   if (LV.getType()->isAtomicType()) {
1476     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1477   } else {
1478     AO = llvm::AtomicOrdering::Acquire;
1479     IsVolatile = true;
1480   }
1481   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1482 }
1483
1484 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1485                                   bool AsValue, llvm::AtomicOrdering AO,
1486                                   bool IsVolatile) {
1487   // Check whether we should use a library call.
1488   if (shouldUseLibcall()) {
1489     Address TempAddr = Address::invalid();
1490     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1491       assert(getEvaluationKind() == TEK_Aggregate);
1492       TempAddr = ResultSlot.getAddress();
1493     } else
1494       TempAddr = CreateTempAlloca();
1495
1496     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1497
1498     // Okay, turn that back into the original value or whole atomic (for
1499     // non-simple lvalues) type.
1500     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1501   }
1502
1503   // Okay, we're doing this natively.
1504   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1505
1506   // If we're ignoring an aggregate return, don't do anything.
1507   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1508     return RValue::getAggregate(Address::invalid(), false);
1509
1510   // Okay, turn that back into the original value or atomic (for non-simple
1511   // lvalues) type.
1512   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1513 }
1514
1515 /// Emit a load from an l-value of atomic type.  Note that the r-value
1516 /// we produce is an r-value of the atomic *value* type.
1517 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1518                                        llvm::AtomicOrdering AO, bool IsVolatile,
1519                                        AggValueSlot resultSlot) {
1520   AtomicInfo Atomics(*this, src);
1521   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1522                                 IsVolatile);
1523 }
1524
1525 /// Copy an r-value into memory as part of storing to an atomic type.
1526 /// This needs to create a bit-pattern suitable for atomic operations.
1527 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1528   assert(LVal.isSimple());
1529   // If we have an r-value, the rvalue should be of the atomic type,
1530   // which means that the caller is responsible for having zeroed
1531   // any padding.  Just do an aggregate copy of that type.
1532   if (rvalue.isAggregate()) {
1533     LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1534     LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1535                                     getAtomicType());
1536     bool IsVolatile = rvalue.isVolatileQualified() ||
1537                       LVal.isVolatileQualified();
1538     CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1539                           AggValueSlot::DoesNotOverlap, IsVolatile);
1540     return;
1541   }
1542
1543   // Okay, otherwise we're copying stuff.
1544
1545   // Zero out the buffer if necessary.
1546   emitMemSetZeroIfNecessary();
1547
1548   // Drill past the padding if present.
1549   LValue TempLVal = projectValue();
1550
1551   // Okay, store the rvalue in.
1552   if (rvalue.isScalar()) {
1553     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1554   } else {
1555     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1556   }
1557 }
1558
1559
1560 /// Materialize an r-value into memory for the purposes of storing it
1561 /// to an atomic type.
1562 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1563   // Aggregate r-values are already in memory, and EmitAtomicStore
1564   // requires them to be values of the atomic type.
1565   if (rvalue.isAggregate())
1566     return rvalue.getAggregateAddress();
1567
1568   // Otherwise, make a temporary and materialize into it.
1569   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1570   AtomicInfo Atomics(CGF, TempLV);
1571   Atomics.emitCopyIntoMemory(rvalue);
1572   return TempLV.getAddress();
1573 }
1574
1575 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1576   // If we've got a scalar value of the right size, try to avoid going
1577   // through memory.
1578   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1579     llvm::Value *Value = RVal.getScalarVal();
1580     if (isa<llvm::IntegerType>(Value->getType()))
1581       return CGF.EmitToMemory(Value, ValueTy);
1582     else {
1583       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1584           CGF.getLLVMContext(),
1585           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1586       if (isa<llvm::PointerType>(Value->getType()))
1587         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1588       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1589         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1590     }
1591   }
1592   // Otherwise, we need to go through memory.
1593   // Put the r-value in memory.
1594   Address Addr = materializeRValue(RVal);
1595
1596   // Cast the temporary to the atomic int type and pull a value out.
1597   Addr = emitCastToAtomicIntPointer(Addr);
1598   return CGF.Builder.CreateLoad(Addr);
1599 }
1600
1601 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1602     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1603     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1604   // Do the atomic store.
1605   Address Addr = getAtomicAddressAsAtomicIntPointer();
1606   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1607                                                ExpectedVal, DesiredVal,
1608                                                Success, Failure);
1609   // Other decoration.
1610   Inst->setVolatile(LVal.isVolatileQualified());
1611   Inst->setWeak(IsWeak);
1612
1613   // Okay, turn that back into the original value type.
1614   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1615   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1616   return std::make_pair(PreviousVal, SuccessFailureVal);
1617 }
1618
1619 llvm::Value *
1620 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1621                                              llvm::Value *DesiredAddr,
1622                                              llvm::AtomicOrdering Success,
1623                                              llvm::AtomicOrdering Failure) {
1624   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1625   // void *desired, int success, int failure);
1626   CallArgList Args;
1627   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1628   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1629            CGF.getContext().VoidPtrTy);
1630   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1631            CGF.getContext().VoidPtrTy);
1632   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1633            CGF.getContext().VoidPtrTy);
1634   Args.add(RValue::get(
1635                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1636            CGF.getContext().IntTy);
1637   Args.add(RValue::get(
1638                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1639            CGF.getContext().IntTy);
1640   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1641                                               CGF.getContext().BoolTy, Args);
1642
1643   return SuccessFailureRVal.getScalarVal();
1644 }
1645
1646 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1647     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1648     llvm::AtomicOrdering Failure, bool IsWeak) {
1649   if (isStrongerThan(Failure, Success))
1650     // Don't assert on undefined behavior "failure argument shall be no stronger
1651     // than the success argument".
1652     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1653
1654   // Check whether we should use a library call.
1655   if (shouldUseLibcall()) {
1656     // Produce a source address.
1657     Address ExpectedAddr = materializeRValue(Expected);
1658     Address DesiredAddr = materializeRValue(Desired);
1659     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1660                                                  DesiredAddr.getPointer(),
1661                                                  Success, Failure);
1662     return std::make_pair(
1663         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1664                                   SourceLocation(), /*AsValue=*/false),
1665         Res);
1666   }
1667
1668   // If we've got a scalar value of the right size, try to avoid going
1669   // through memory.
1670   auto *ExpectedVal = convertRValueToInt(Expected);
1671   auto *DesiredVal = convertRValueToInt(Desired);
1672   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1673                                          Failure, IsWeak);
1674   return std::make_pair(
1675       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1676                                 SourceLocation(), /*AsValue=*/false),
1677       Res.second);
1678 }
1679
1680 static void
1681 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1682                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1683                       Address DesiredAddr) {
1684   RValue UpRVal;
1685   LValue AtomicLVal = Atomics.getAtomicLValue();
1686   LValue DesiredLVal;
1687   if (AtomicLVal.isSimple()) {
1688     UpRVal = OldRVal;
1689     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1690   } else {
1691     // Build new lvalue for temp address
1692     Address Ptr = Atomics.materializeRValue(OldRVal);
1693     LValue UpdateLVal;
1694     if (AtomicLVal.isBitField()) {
1695       UpdateLVal =
1696           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1697                                AtomicLVal.getType(),
1698                                AtomicLVal.getBaseInfo(),
1699                                AtomicLVal.getTBAAInfo());
1700       DesiredLVal =
1701           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1702                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1703                                AtomicLVal.getTBAAInfo());
1704     } else if (AtomicLVal.isVectorElt()) {
1705       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1706                                          AtomicLVal.getType(),
1707                                          AtomicLVal.getBaseInfo(),
1708                                          AtomicLVal.getTBAAInfo());
1709       DesiredLVal = LValue::MakeVectorElt(
1710           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1711           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1712     } else {
1713       assert(AtomicLVal.isExtVectorElt());
1714       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1715                                             AtomicLVal.getType(),
1716                                             AtomicLVal.getBaseInfo(),
1717                                             AtomicLVal.getTBAAInfo());
1718       DesiredLVal = LValue::MakeExtVectorElt(
1719           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1720           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1721     }
1722     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1723   }
1724   // Store new value in the corresponding memory area
1725   RValue NewRVal = UpdateOp(UpRVal);
1726   if (NewRVal.isScalar()) {
1727     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1728   } else {
1729     assert(NewRVal.isComplex());
1730     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1731                            /*isInit=*/false);
1732   }
1733 }
1734
1735 void AtomicInfo::EmitAtomicUpdateLibcall(
1736     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1737     bool IsVolatile) {
1738   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1739
1740   Address ExpectedAddr = CreateTempAlloca();
1741
1742   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1743   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1744   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1745   CGF.EmitBlock(ContBB);
1746   Address DesiredAddr = CreateTempAlloca();
1747   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1748       requiresMemSetZero(getAtomicAddress().getElementType())) {
1749     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1750     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1751   }
1752   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1753                                            AggValueSlot::ignored(),
1754                                            SourceLocation(), /*AsValue=*/false);
1755   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1756   auto *Res =
1757       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1758                                        DesiredAddr.getPointer(),
1759                                        AO, Failure);
1760   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1761   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1762 }
1763
1764 void AtomicInfo::EmitAtomicUpdateOp(
1765     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1766     bool IsVolatile) {
1767   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1768
1769   // Do the atomic load.
1770   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1771   // For non-simple lvalues perform compare-and-swap procedure.
1772   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1773   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1774   auto *CurBB = CGF.Builder.GetInsertBlock();
1775   CGF.EmitBlock(ContBB);
1776   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1777                                              /*NumReservedValues=*/2);
1778   PHI->addIncoming(OldVal, CurBB);
1779   Address NewAtomicAddr = CreateTempAlloca();
1780   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1781   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1782       requiresMemSetZero(getAtomicAddress().getElementType())) {
1783     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1784   }
1785   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1786                                            SourceLocation(), /*AsValue=*/false);
1787   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1788   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1789   // Try to write new value using cmpxchg operation
1790   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1791   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1792   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1793   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1794 }
1795
1796 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1797                                   RValue UpdateRVal, Address DesiredAddr) {
1798   LValue AtomicLVal = Atomics.getAtomicLValue();
1799   LValue DesiredLVal;
1800   // Build new lvalue for temp address
1801   if (AtomicLVal.isBitField()) {
1802     DesiredLVal =
1803         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1804                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1805                              AtomicLVal.getTBAAInfo());
1806   } else if (AtomicLVal.isVectorElt()) {
1807     DesiredLVal =
1808         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1809                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1810                               AtomicLVal.getTBAAInfo());
1811   } else {
1812     assert(AtomicLVal.isExtVectorElt());
1813     DesiredLVal = LValue::MakeExtVectorElt(
1814         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1815         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1816   }
1817   // Store new value in the corresponding memory area
1818   assert(UpdateRVal.isScalar());
1819   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1820 }
1821
1822 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1823                                          RValue UpdateRVal, bool IsVolatile) {
1824   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1825
1826   Address ExpectedAddr = CreateTempAlloca();
1827
1828   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1829   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1830   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1831   CGF.EmitBlock(ContBB);
1832   Address DesiredAddr = CreateTempAlloca();
1833   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1834       requiresMemSetZero(getAtomicAddress().getElementType())) {
1835     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1836     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1837   }
1838   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1839   auto *Res =
1840       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1841                                        DesiredAddr.getPointer(),
1842                                        AO, Failure);
1843   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1844   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1845 }
1846
1847 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1848                                     bool IsVolatile) {
1849   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1850
1851   // Do the atomic load.
1852   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1853   // For non-simple lvalues perform compare-and-swap procedure.
1854   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1855   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1856   auto *CurBB = CGF.Builder.GetInsertBlock();
1857   CGF.EmitBlock(ContBB);
1858   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1859                                              /*NumReservedValues=*/2);
1860   PHI->addIncoming(OldVal, CurBB);
1861   Address NewAtomicAddr = CreateTempAlloca();
1862   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1863   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1864       requiresMemSetZero(getAtomicAddress().getElementType())) {
1865     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1866   }
1867   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1868   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1869   // Try to write new value using cmpxchg operation
1870   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1871   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1872   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1873   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1874 }
1875
1876 void AtomicInfo::EmitAtomicUpdate(
1877     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1878     bool IsVolatile) {
1879   if (shouldUseLibcall()) {
1880     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1881   } else {
1882     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1883   }
1884 }
1885
1886 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1887                                   bool IsVolatile) {
1888   if (shouldUseLibcall()) {
1889     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1890   } else {
1891     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1892   }
1893 }
1894
1895 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1896                                       bool isInit) {
1897   bool IsVolatile = lvalue.isVolatileQualified();
1898   llvm::AtomicOrdering AO;
1899   if (lvalue.getType()->isAtomicType()) {
1900     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1901   } else {
1902     AO = llvm::AtomicOrdering::Release;
1903     IsVolatile = true;
1904   }
1905   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1906 }
1907
1908 /// Emit a store to an l-value of atomic type.
1909 ///
1910 /// Note that the r-value is expected to be an r-value *of the atomic
1911 /// type*; this means that for aggregate r-values, it should include
1912 /// storage for any padding that was necessary.
1913 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1914                                       llvm::AtomicOrdering AO, bool IsVolatile,
1915                                       bool isInit) {
1916   // If this is an aggregate r-value, it should agree in type except
1917   // maybe for address-space qualification.
1918   assert(!rvalue.isAggregate() ||
1919          rvalue.getAggregateAddress().getElementType()
1920            == dest.getAddress().getElementType());
1921
1922   AtomicInfo atomics(*this, dest);
1923   LValue LVal = atomics.getAtomicLValue();
1924
1925   // If this is an initialization, just put the value there normally.
1926   if (LVal.isSimple()) {
1927     if (isInit) {
1928       atomics.emitCopyIntoMemory(rvalue);
1929       return;
1930     }
1931
1932     // Check whether we should use a library call.
1933     if (atomics.shouldUseLibcall()) {
1934       // Produce a source address.
1935       Address srcAddr = atomics.materializeRValue(rvalue);
1936
1937       // void __atomic_store(size_t size, void *mem, void *val, int order)
1938       CallArgList args;
1939       args.add(RValue::get(atomics.getAtomicSizeValue()),
1940                getContext().getSizeType());
1941       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1942                getContext().VoidPtrTy);
1943       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1944                getContext().VoidPtrTy);
1945       args.add(
1946           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1947           getContext().IntTy);
1948       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1949       return;
1950     }
1951
1952     // Okay, we're doing this natively.
1953     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1954
1955     // Do the atomic store.
1956     Address addr =
1957         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1958     intValue = Builder.CreateIntCast(
1959         intValue, addr.getElementType(), /*isSigned=*/false);
1960     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1961
1962     // Initializations don't need to be atomic.
1963     if (!isInit)
1964       store->setAtomic(AO);
1965
1966     // Other decoration.
1967     if (IsVolatile)
1968       store->setVolatile(true);
1969     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1970     return;
1971   }
1972
1973   // Emit simple atomic update operation.
1974   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1975 }
1976
1977 /// Emit a compare-and-exchange op for atomic type.
1978 ///
1979 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1980     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1981     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1982     AggValueSlot Slot) {
1983   // If this is an aggregate r-value, it should agree in type except
1984   // maybe for address-space qualification.
1985   assert(!Expected.isAggregate() ||
1986          Expected.getAggregateAddress().getElementType() ==
1987              Obj.getAddress().getElementType());
1988   assert(!Desired.isAggregate() ||
1989          Desired.getAggregateAddress().getElementType() ==
1990              Obj.getAddress().getElementType());
1991   AtomicInfo Atomics(*this, Obj);
1992
1993   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1994                                            IsWeak);
1995 }
1996
1997 void CodeGenFunction::EmitAtomicUpdate(
1998     LValue LVal, llvm::AtomicOrdering AO,
1999     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2000   AtomicInfo Atomics(*this, LVal);
2001   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2002 }
2003
2004 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2005   AtomicInfo atomics(*this, dest);
2006
2007   switch (atomics.getEvaluationKind()) {
2008   case TEK_Scalar: {
2009     llvm::Value *value = EmitScalarExpr(init);
2010     atomics.emitCopyIntoMemory(RValue::get(value));
2011     return;
2012   }
2013
2014   case TEK_Complex: {
2015     ComplexPairTy value = EmitComplexExpr(init);
2016     atomics.emitCopyIntoMemory(RValue::getComplex(value));
2017     return;
2018   }
2019
2020   case TEK_Aggregate: {
2021     // Fix up the destination if the initializer isn't an expression
2022     // of atomic type.
2023     bool Zeroed = false;
2024     if (!init->getType()->isAtomicType()) {
2025       Zeroed = atomics.emitMemSetZeroIfNecessary();
2026       dest = atomics.projectValue();
2027     }
2028
2029     // Evaluate the expression directly into the destination.
2030     AggValueSlot slot = AggValueSlot::forLValue(dest,
2031                                         AggValueSlot::IsNotDestructed,
2032                                         AggValueSlot::DoesNotNeedGCBarriers,
2033                                         AggValueSlot::IsNotAliased,
2034                                         AggValueSlot::DoesNotOverlap,
2035                                         Zeroed ? AggValueSlot::IsZeroed :
2036                                                  AggValueSlot::IsNotZeroed);
2037
2038     EmitAggExpr(init, slot);
2039     return;
2040   }
2041   }
2042   llvm_unreachable("bad evaluation kind");
2043 }