1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the code for emitting atomic operations.
12 //===----------------------------------------------------------------------===//
15 #include "CGRecordLayout.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "llvm/IR/DataLayout.h"
21 #include "llvm/IR/Intrinsics.h"
22 #include "llvm/IR/Operator.h"
24 using namespace clang;
25 using namespace CodeGen;
32 uint64_t AtomicSizeInBits;
33 uint64_t ValueSizeInBits;
34 CharUnits AtomicAlign;
36 CharUnits LValueAlign;
37 TypeEvaluationKind EvaluationKind;
42 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
43 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
44 EvaluationKind(TEK_Scalar), UseLibcall(true) {
45 assert(!lvalue.isGlobalReg());
46 ASTContext &C = CGF.getContext();
47 if (lvalue.isSimple()) {
48 AtomicTy = lvalue.getType();
49 if (auto *ATy = AtomicTy->getAs<AtomicType>())
50 ValueTy = ATy->getValueType();
53 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 uint64_t ValueAlignInBits;
56 uint64_t AtomicAlignInBits;
57 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
58 ValueSizeInBits = ValueTI.Width;
59 ValueAlignInBits = ValueTI.Align;
61 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
62 AtomicSizeInBits = AtomicTI.Width;
63 AtomicAlignInBits = AtomicTI.Align;
65 assert(ValueSizeInBits <= AtomicSizeInBits);
66 assert(ValueAlignInBits <= AtomicAlignInBits);
68 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
69 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
70 if (lvalue.getAlignment().isZero())
71 lvalue.setAlignment(AtomicAlign);
74 } else if (lvalue.isBitField()) {
75 ValueTy = lvalue.getType();
76 ValueSizeInBits = C.getTypeSize(ValueTy);
77 auto &OrigBFI = lvalue.getBitFieldInfo();
78 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
79 AtomicSizeInBits = C.toBits(
80 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
81 .alignTo(lvalue.getAlignment()));
82 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
85 lvalue.getAlignment();
86 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
87 VoidPtrAddr, OffsetInChars.getQuantity());
88 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
91 "atomic_bitfield_base");
94 BFI.StorageSize = AtomicSizeInBits;
95 BFI.StorageOffset += OffsetInChars;
96 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
97 BFI, lvalue.getType(),
98 lvalue.getBaseInfo());
99 LVal.setTBAAInfo(lvalue.getTBAAInfo());
100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101 if (AtomicTy.isNull()) {
104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
106 /*IndexTypeQuals=*/0);
108 AtomicAlign = ValueAlign = lvalue.getAlignment();
109 } else if (lvalue.isVectorElt()) {
110 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
111 ValueSizeInBits = C.getTypeSize(ValueTy);
112 AtomicTy = lvalue.getType();
113 AtomicSizeInBits = C.getTypeSize(AtomicTy);
114 AtomicAlign = ValueAlign = lvalue.getAlignment();
117 assert(lvalue.isExtVectorElt());
118 ValueTy = lvalue.getType();
119 ValueSizeInBits = C.getTypeSize(ValueTy);
120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121 lvalue.getType(), lvalue.getExtVectorAddress()
122 .getElementType()->getVectorNumElements());
123 AtomicSizeInBits = C.getTypeSize(AtomicTy);
124 AtomicAlign = ValueAlign = lvalue.getAlignment();
127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131 QualType getAtomicType() const { return AtomicTy; }
132 QualType getValueType() const { return ValueTy; }
133 CharUnits getAtomicAlignment() const { return AtomicAlign; }
134 CharUnits getValueAlignment() const { return ValueAlign; }
135 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138 bool shouldUseLibcall() const { return UseLibcall; }
139 const LValue &getAtomicLValue() const { return LVal; }
140 llvm::Value *getAtomicPointer() const {
142 return LVal.getPointer();
143 else if (LVal.isBitField())
144 return LVal.getBitFieldPointer();
145 else if (LVal.isVectorElt())
146 return LVal.getVectorPointer();
147 assert(LVal.isExtVectorElt());
148 return LVal.getExtVectorPointer();
150 Address getAtomicAddress() const {
151 return Address(getAtomicPointer(), getAtomicAlignment());
154 Address getAtomicAddressAsAtomicIntPointer() const {
155 return emitCastToAtomicIntPointer(getAtomicAddress());
158 /// Is the atomic size larger than the underlying value type?
160 /// Note that the absence of padding does not mean that atomic
161 /// objects are completely interchangeable with non-atomic
162 /// objects: we might have promoted the alignment of a type
163 /// without making it bigger.
164 bool hasPadding() const {
165 return (ValueSizeInBits != AtomicSizeInBits);
168 bool emitMemSetZeroIfNecessary() const;
170 llvm::Value *getAtomicSizeValue() const {
171 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
172 return CGF.CGM.getSize(size);
175 /// Cast the given pointer to an integer pointer suitable for atomic
176 /// operations if the source.
177 Address emitCastToAtomicIntPointer(Address Addr) const;
179 /// If Addr is compatible with the iN that will be used for an atomic
180 /// operation, bitcast it. Otherwise, create a temporary that is suitable
181 /// and copy the value across.
182 Address convertToAtomicIntPointer(Address Addr) const;
184 /// Turn an atomic-layout object into an r-value.
185 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
186 SourceLocation loc, bool AsValue) const;
188 /// \brief Converts a rvalue to integer value.
189 llvm::Value *convertRValueToInt(RValue RVal) const;
191 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
192 AggValueSlot ResultSlot,
193 SourceLocation Loc, bool AsValue) const;
195 /// Copy an atomic r-value into atomic-layout memory.
196 void emitCopyIntoMemory(RValue rvalue) const;
198 /// Project an l-value down to the value field.
199 LValue projectValue() const {
200 assert(LVal.isSimple());
201 Address addr = getAtomicAddress();
203 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
205 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
206 LVal.getBaseInfo(), LVal.getTBAAInfo());
209 /// \brief Emits atomic load.
210 /// \returns Loaded value.
211 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
212 bool AsValue, llvm::AtomicOrdering AO,
215 /// \brief Emits atomic compare-and-exchange sequence.
216 /// \param Expected Expected value.
217 /// \param Desired Desired value.
218 /// \param Success Atomic ordering for success operation.
219 /// \param Failure Atomic ordering for failed operation.
220 /// \param IsWeak true if atomic operation is weak, false otherwise.
221 /// \returns Pair of values: previous value from storage (value type) and
222 /// boolean flag (i1 type) with true if success and false otherwise.
223 std::pair<RValue, llvm::Value *>
224 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
225 llvm::AtomicOrdering Success =
226 llvm::AtomicOrdering::SequentiallyConsistent,
227 llvm::AtomicOrdering Failure =
228 llvm::AtomicOrdering::SequentiallyConsistent,
229 bool IsWeak = false);
231 /// \brief Emits atomic update.
232 /// \param AO Atomic ordering.
233 /// \param UpdateOp Update operation for the current lvalue.
234 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
235 const llvm::function_ref<RValue(RValue)> &UpdateOp,
237 /// \brief Emits atomic update.
238 /// \param AO Atomic ordering.
239 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
242 /// Materialize an atomic r-value in atomic-layout memory.
243 Address materializeRValue(RValue rvalue) const;
245 /// \brief Creates temp alloca for intermediate operations on atomic value.
246 Address CreateTempAlloca() const;
248 bool requiresMemSetZero(llvm::Type *type) const;
251 /// \brief Emits atomic load as a libcall.
252 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
253 llvm::AtomicOrdering AO, bool IsVolatile);
254 /// \brief Emits atomic load as LLVM instruction.
255 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
256 /// \brief Emits atomic compare-and-exchange op as a libcall.
257 llvm::Value *EmitAtomicCompareExchangeLibcall(
258 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
259 llvm::AtomicOrdering Success =
260 llvm::AtomicOrdering::SequentiallyConsistent,
261 llvm::AtomicOrdering Failure =
262 llvm::AtomicOrdering::SequentiallyConsistent);
263 /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
264 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
265 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
266 llvm::AtomicOrdering Success =
267 llvm::AtomicOrdering::SequentiallyConsistent,
268 llvm::AtomicOrdering Failure =
269 llvm::AtomicOrdering::SequentiallyConsistent,
270 bool IsWeak = false);
271 /// \brief Emit atomic update as libcalls.
273 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
274 const llvm::function_ref<RValue(RValue)> &UpdateOp,
276 /// \brief Emit atomic update as LLVM instructions.
277 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
278 const llvm::function_ref<RValue(RValue)> &UpdateOp,
280 /// \brief Emit atomic update as libcalls.
281 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283 /// \brief Emit atomic update as LLVM instructions.
284 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
289 Address AtomicInfo::CreateTempAlloca() const {
290 Address TempAlloca = CGF.CreateMemTemp(
291 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293 getAtomicAlignment(),
295 // Cast to pointer to value type for bitfields.
296 if (LVal.isBitField())
297 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
298 TempAlloca, getAtomicAddress().getType());
302 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
306 const CGFunctionInfo &fnInfo =
307 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
308 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
309 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
310 auto callee = CGCallee::forDirect(fn);
311 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
314 /// Does a store of the given IR type modify the full expected width?
315 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
316 uint64_t expectedSize) {
317 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
320 /// Does the atomic type require memsetting to zero before initialization?
322 /// The IR type is provided as a way of making certain queries faster.
323 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
324 // If the atomic type has size padding, we definitely need a memset.
325 if (hasPadding()) return true;
327 // Otherwise, do some simple heuristics to try to avoid it:
328 switch (getEvaluationKind()) {
329 // For scalars and complexes, check whether the store size of the
330 // type uses the full size.
332 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
334 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
335 AtomicSizeInBits / 2);
337 // Padding in structs has an undefined bit pattern. User beware.
341 llvm_unreachable("bad evaluation kind");
344 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
345 assert(LVal.isSimple());
346 llvm::Value *addr = LVal.getPointer();
347 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
350 CGF.Builder.CreateMemSet(
351 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
352 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
353 LVal.getAlignment().getQuantity());
357 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
358 Address Dest, Address Ptr,
359 Address Val1, Address Val2,
361 llvm::AtomicOrdering SuccessOrder,
362 llvm::AtomicOrdering FailureOrder) {
363 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
364 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
367 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
368 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
369 Pair->setVolatile(E->isVolatile());
370 Pair->setWeak(IsWeak);
372 // Cmp holds the result of the compare-exchange operation: true on success,
374 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
375 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
377 // This basic block is used to hold the store instruction if the operation
379 llvm::BasicBlock *StoreExpectedBB =
380 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
382 // This basic block is the exit point of the operation, we should end up
383 // here regardless of whether or not the operation succeeded.
384 llvm::BasicBlock *ContinueBB =
385 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
387 // Update Expected if Expected isn't equal to Old, otherwise branch to the
389 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
391 CGF.Builder.SetInsertPoint(StoreExpectedBB);
392 // Update the memory at Expected with Old's value.
393 CGF.Builder.CreateStore(Old, Val1);
394 // Finally, branch to the exit point.
395 CGF.Builder.CreateBr(ContinueBB);
397 CGF.Builder.SetInsertPoint(ContinueBB);
398 // Update the memory at Dest with Cmp's value.
399 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
402 /// Given an ordering required on success, emit all possible cmpxchg
403 /// instructions to cope with the provided (but possibly only dynamically known)
405 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
406 bool IsWeak, Address Dest, Address Ptr,
407 Address Val1, Address Val2,
408 llvm::Value *FailureOrderVal,
410 llvm::AtomicOrdering SuccessOrder) {
411 llvm::AtomicOrdering FailureOrder;
412 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
413 auto FOS = FO->getSExtValue();
414 if (!llvm::isValidAtomicOrderingCABI(FOS))
415 FailureOrder = llvm::AtomicOrdering::Monotonic;
417 switch ((llvm::AtomicOrderingCABI)FOS) {
418 case llvm::AtomicOrderingCABI::relaxed:
419 case llvm::AtomicOrderingCABI::release:
420 case llvm::AtomicOrderingCABI::acq_rel:
421 FailureOrder = llvm::AtomicOrdering::Monotonic;
423 case llvm::AtomicOrderingCABI::consume:
424 case llvm::AtomicOrderingCABI::acquire:
425 FailureOrder = llvm::AtomicOrdering::Acquire;
427 case llvm::AtomicOrderingCABI::seq_cst:
428 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
431 if (isStrongerThan(FailureOrder, SuccessOrder)) {
432 // Don't assert on undefined behavior "failure argument shall be no
433 // stronger than the success argument".
435 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
437 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
442 // Create all the relevant BB's
443 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
445 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
446 if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
447 SuccessOrder != llvm::AtomicOrdering::Release)
448 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
449 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
450 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
452 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
454 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
456 // Emit all the different atomics
458 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
459 // doesn't matter unless someone is crazy enough to use something that
460 // doesn't fold to a constant for the ordering.
461 CGF.Builder.SetInsertPoint(MonotonicBB);
462 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
463 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic);
464 CGF.Builder.CreateBr(ContBB);
467 CGF.Builder.SetInsertPoint(AcquireBB);
468 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
469 Size, SuccessOrder, llvm::AtomicOrdering::Acquire);
470 CGF.Builder.CreateBr(ContBB);
471 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
477 CGF.Builder.SetInsertPoint(SeqCstBB);
478 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
479 llvm::AtomicOrdering::SequentiallyConsistent);
480 CGF.Builder.CreateBr(ContBB);
481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
485 CGF.Builder.SetInsertPoint(ContBB);
488 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
489 Address Ptr, Address Val1, Address Val2,
490 llvm::Value *IsWeak, llvm::Value *FailureOrder,
491 uint64_t Size, llvm::AtomicOrdering Order) {
492 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
493 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
495 switch (E->getOp()) {
496 case AtomicExpr::AO__c11_atomic_init:
497 llvm_unreachable("Already handled!");
499 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
500 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
501 FailureOrder, Size, Order);
503 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
504 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
505 FailureOrder, Size, Order);
507 case AtomicExpr::AO__atomic_compare_exchange:
508 case AtomicExpr::AO__atomic_compare_exchange_n: {
509 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
510 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
511 Val1, Val2, FailureOrder, Size, Order);
513 // Create all the relevant BB's
514 llvm::BasicBlock *StrongBB =
515 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
516 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
517 llvm::BasicBlock *ContBB =
518 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
520 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
521 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
523 CGF.Builder.SetInsertPoint(StrongBB);
524 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
525 FailureOrder, Size, Order);
526 CGF.Builder.CreateBr(ContBB);
528 CGF.Builder.SetInsertPoint(WeakBB);
529 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
530 FailureOrder, Size, Order);
531 CGF.Builder.CreateBr(ContBB);
533 CGF.Builder.SetInsertPoint(ContBB);
537 case AtomicExpr::AO__c11_atomic_load:
538 case AtomicExpr::AO__atomic_load_n:
539 case AtomicExpr::AO__atomic_load: {
540 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
541 Load->setAtomic(Order);
542 Load->setVolatile(E->isVolatile());
543 CGF.Builder.CreateStore(Load, Dest);
547 case AtomicExpr::AO__c11_atomic_store:
548 case AtomicExpr::AO__atomic_store:
549 case AtomicExpr::AO__atomic_store_n: {
550 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
551 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
552 Store->setAtomic(Order);
553 Store->setVolatile(E->isVolatile());
557 case AtomicExpr::AO__c11_atomic_exchange:
558 case AtomicExpr::AO__atomic_exchange_n:
559 case AtomicExpr::AO__atomic_exchange:
560 Op = llvm::AtomicRMWInst::Xchg;
563 case AtomicExpr::AO__atomic_add_fetch:
564 PostOp = llvm::Instruction::Add;
566 case AtomicExpr::AO__c11_atomic_fetch_add:
567 case AtomicExpr::AO__atomic_fetch_add:
568 Op = llvm::AtomicRMWInst::Add;
571 case AtomicExpr::AO__atomic_sub_fetch:
572 PostOp = llvm::Instruction::Sub;
574 case AtomicExpr::AO__c11_atomic_fetch_sub:
575 case AtomicExpr::AO__atomic_fetch_sub:
576 Op = llvm::AtomicRMWInst::Sub;
579 case AtomicExpr::AO__atomic_and_fetch:
580 PostOp = llvm::Instruction::And;
582 case AtomicExpr::AO__c11_atomic_fetch_and:
583 case AtomicExpr::AO__atomic_fetch_and:
584 Op = llvm::AtomicRMWInst::And;
587 case AtomicExpr::AO__atomic_or_fetch:
588 PostOp = llvm::Instruction::Or;
590 case AtomicExpr::AO__c11_atomic_fetch_or:
591 case AtomicExpr::AO__atomic_fetch_or:
592 Op = llvm::AtomicRMWInst::Or;
595 case AtomicExpr::AO__atomic_xor_fetch:
596 PostOp = llvm::Instruction::Xor;
598 case AtomicExpr::AO__c11_atomic_fetch_xor:
599 case AtomicExpr::AO__atomic_fetch_xor:
600 Op = llvm::AtomicRMWInst::Xor;
603 case AtomicExpr::AO__atomic_nand_fetch:
604 PostOp = llvm::Instruction::And; // the NOT is special cased below
606 case AtomicExpr::AO__atomic_fetch_nand:
607 Op = llvm::AtomicRMWInst::Nand;
611 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
612 llvm::AtomicRMWInst *RMWI =
613 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
614 RMWI->setVolatile(E->isVolatile());
616 // For __atomic_*_fetch operations, perform the operation again to
617 // determine the value which was written.
618 llvm::Value *Result = RMWI;
620 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
621 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
622 Result = CGF.Builder.CreateNot(Result);
623 CGF.Builder.CreateStore(Result, Dest);
626 // This function emits any expression (scalar, complex, or aggregate)
627 // into a temporary alloca.
629 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
630 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
631 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
637 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
638 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
639 SourceLocation Loc, CharUnits SizeInChars) {
640 if (UseOptimizedLibcall) {
641 // Load value and pass it to the function directly.
642 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
643 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
645 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
646 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
647 SizeInBits)->getPointerTo();
648 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
649 Val = CGF.EmitLoadOfScalar(Ptr, false,
650 CGF.getContext().getPointerType(ValTy),
652 // Coerce the value into an appropriately sized integer type.
653 Args.add(RValue::get(Val), ValTy);
655 // Non-optimized functions always take a reference.
656 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
657 CGF.getContext().VoidPtrTy);
661 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
662 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
663 QualType MemTy = AtomicTy;
664 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
665 MemTy = AT->getValueType();
666 CharUnits sizeChars, alignChars;
667 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
668 uint64_t Size = sizeChars.getQuantity();
669 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
670 bool UseLibcall = (sizeChars != alignChars ||
671 getContext().toBits(sizeChars) > MaxInlineWidthInBits);
673 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
675 Address Val1 = Address::invalid();
676 Address Val2 = Address::invalid();
677 Address Dest = Address::invalid();
678 Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
680 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
681 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
682 EmitAtomicInit(E->getVal1(), lvalue);
683 return RValue::get(nullptr);
686 llvm::Value *Order = EmitScalarExpr(E->getOrder());
688 switch (E->getOp()) {
689 case AtomicExpr::AO__c11_atomic_init:
690 llvm_unreachable("Already handled above with EmitAtomicInit!");
692 case AtomicExpr::AO__c11_atomic_load:
693 case AtomicExpr::AO__atomic_load_n:
696 case AtomicExpr::AO__atomic_load:
697 Dest = EmitPointerWithAlignment(E->getVal1());
700 case AtomicExpr::AO__atomic_store:
701 Val1 = EmitPointerWithAlignment(E->getVal1());
704 case AtomicExpr::AO__atomic_exchange:
705 Val1 = EmitPointerWithAlignment(E->getVal1());
706 Dest = EmitPointerWithAlignment(E->getVal2());
709 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
710 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
711 case AtomicExpr::AO__atomic_compare_exchange_n:
712 case AtomicExpr::AO__atomic_compare_exchange:
713 Val1 = EmitPointerWithAlignment(E->getVal1());
714 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
715 Val2 = EmitPointerWithAlignment(E->getVal2());
717 Val2 = EmitValToTemp(*this, E->getVal2());
718 OrderFail = EmitScalarExpr(E->getOrderFail());
719 if (E->getNumSubExprs() == 6)
720 IsWeak = EmitScalarExpr(E->getWeak());
723 case AtomicExpr::AO__c11_atomic_fetch_add:
724 case AtomicExpr::AO__c11_atomic_fetch_sub:
725 if (MemTy->isPointerType()) {
726 // For pointer arithmetic, we're required to do a bit of math:
727 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
728 // ... but only for the C11 builtins. The GNU builtins expect the
729 // user to multiply by sizeof(T).
730 QualType Val1Ty = E->getVal1()->getType();
731 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
732 CharUnits PointeeIncAmt =
733 getContext().getTypeSizeInChars(MemTy->getPointeeType());
734 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
735 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
737 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
741 case AtomicExpr::AO__atomic_fetch_add:
742 case AtomicExpr::AO__atomic_fetch_sub:
743 case AtomicExpr::AO__atomic_add_fetch:
744 case AtomicExpr::AO__atomic_sub_fetch:
745 case AtomicExpr::AO__c11_atomic_store:
746 case AtomicExpr::AO__c11_atomic_exchange:
747 case AtomicExpr::AO__atomic_store_n:
748 case AtomicExpr::AO__atomic_exchange_n:
749 case AtomicExpr::AO__c11_atomic_fetch_and:
750 case AtomicExpr::AO__c11_atomic_fetch_or:
751 case AtomicExpr::AO__c11_atomic_fetch_xor:
752 case AtomicExpr::AO__atomic_fetch_and:
753 case AtomicExpr::AO__atomic_fetch_or:
754 case AtomicExpr::AO__atomic_fetch_xor:
755 case AtomicExpr::AO__atomic_fetch_nand:
756 case AtomicExpr::AO__atomic_and_fetch:
757 case AtomicExpr::AO__atomic_or_fetch:
758 case AtomicExpr::AO__atomic_xor_fetch:
759 case AtomicExpr::AO__atomic_nand_fetch:
760 Val1 = EmitValToTemp(*this, E->getVal1());
764 QualType RValTy = E->getType().getUnqualifiedType();
766 // The inlined atomics only function on iN types, where N is a power of 2. We
767 // need to make sure (via temporaries if necessary) that all incoming values
769 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
770 AtomicInfo Atomics(*this, AtomicVal);
772 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
773 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
774 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
776 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
777 else if (E->isCmpXChg())
778 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
779 else if (!RValTy->isVoidType())
780 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
782 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
784 bool UseOptimizedLibcall = false;
785 switch (E->getOp()) {
786 case AtomicExpr::AO__c11_atomic_init:
787 llvm_unreachable("Already handled above with EmitAtomicInit!");
789 case AtomicExpr::AO__c11_atomic_fetch_add:
790 case AtomicExpr::AO__atomic_fetch_add:
791 case AtomicExpr::AO__c11_atomic_fetch_and:
792 case AtomicExpr::AO__atomic_fetch_and:
793 case AtomicExpr::AO__c11_atomic_fetch_or:
794 case AtomicExpr::AO__atomic_fetch_or:
795 case AtomicExpr::AO__atomic_fetch_nand:
796 case AtomicExpr::AO__c11_atomic_fetch_sub:
797 case AtomicExpr::AO__atomic_fetch_sub:
798 case AtomicExpr::AO__c11_atomic_fetch_xor:
799 case AtomicExpr::AO__atomic_fetch_xor:
800 case AtomicExpr::AO__atomic_add_fetch:
801 case AtomicExpr::AO__atomic_and_fetch:
802 case AtomicExpr::AO__atomic_nand_fetch:
803 case AtomicExpr::AO__atomic_or_fetch:
804 case AtomicExpr::AO__atomic_sub_fetch:
805 case AtomicExpr::AO__atomic_xor_fetch:
806 // For these, only library calls for certain sizes exist.
807 UseOptimizedLibcall = true;
810 case AtomicExpr::AO__c11_atomic_load:
811 case AtomicExpr::AO__c11_atomic_store:
812 case AtomicExpr::AO__c11_atomic_exchange:
813 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
814 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
815 case AtomicExpr::AO__atomic_load_n:
816 case AtomicExpr::AO__atomic_load:
817 case AtomicExpr::AO__atomic_store_n:
818 case AtomicExpr::AO__atomic_store:
819 case AtomicExpr::AO__atomic_exchange_n:
820 case AtomicExpr::AO__atomic_exchange:
821 case AtomicExpr::AO__atomic_compare_exchange_n:
822 case AtomicExpr::AO__atomic_compare_exchange:
823 // Only use optimized library calls for sizes for which they exist.
824 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
825 UseOptimizedLibcall = true;
830 if (!UseOptimizedLibcall) {
831 // For non-optimized library calls, the size is the first parameter
832 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
833 getContext().getSizeType());
835 // Atomic address is the first or second parameter
836 Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())),
837 getContext().VoidPtrTy);
839 std::string LibCallName;
840 QualType LoweredMemTy =
841 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
843 bool HaveRetTy = false;
844 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
845 switch (E->getOp()) {
846 case AtomicExpr::AO__c11_atomic_init:
847 llvm_unreachable("Already handled!");
849 // There is only one libcall for compare an exchange, because there is no
850 // optimisation benefit possible from a libcall version of a weak compare
852 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
853 // void *desired, int success, int failure)
854 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
855 // int success, int failure)
856 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
857 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
858 case AtomicExpr::AO__atomic_compare_exchange:
859 case AtomicExpr::AO__atomic_compare_exchange_n:
860 LibCallName = "__atomic_compare_exchange";
861 RetTy = getContext().BoolTy;
863 Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())),
864 getContext().VoidPtrTy);
865 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
866 MemTy, E->getExprLoc(), sizeChars);
867 Args.add(RValue::get(Order), getContext().IntTy);
870 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
872 // T __atomic_exchange_N(T *mem, T val, int order)
873 case AtomicExpr::AO__c11_atomic_exchange:
874 case AtomicExpr::AO__atomic_exchange_n:
875 case AtomicExpr::AO__atomic_exchange:
876 LibCallName = "__atomic_exchange";
877 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
878 MemTy, E->getExprLoc(), sizeChars);
880 // void __atomic_store(size_t size, void *mem, void *val, int order)
881 // void __atomic_store_N(T *mem, T val, int order)
882 case AtomicExpr::AO__c11_atomic_store:
883 case AtomicExpr::AO__atomic_store:
884 case AtomicExpr::AO__atomic_store_n:
885 LibCallName = "__atomic_store";
886 RetTy = getContext().VoidTy;
888 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
889 MemTy, E->getExprLoc(), sizeChars);
891 // void __atomic_load(size_t size, void *mem, void *return, int order)
892 // T __atomic_load_N(T *mem, int order)
893 case AtomicExpr::AO__c11_atomic_load:
894 case AtomicExpr::AO__atomic_load:
895 case AtomicExpr::AO__atomic_load_n:
896 LibCallName = "__atomic_load";
898 // T __atomic_add_fetch_N(T *mem, T val, int order)
899 // T __atomic_fetch_add_N(T *mem, T val, int order)
900 case AtomicExpr::AO__atomic_add_fetch:
901 PostOp = llvm::Instruction::Add;
903 case AtomicExpr::AO__c11_atomic_fetch_add:
904 case AtomicExpr::AO__atomic_fetch_add:
905 LibCallName = "__atomic_fetch_add";
906 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
907 LoweredMemTy, E->getExprLoc(), sizeChars);
909 // T __atomic_and_fetch_N(T *mem, T val, int order)
910 // T __atomic_fetch_and_N(T *mem, T val, int order)
911 case AtomicExpr::AO__atomic_and_fetch:
912 PostOp = llvm::Instruction::And;
914 case AtomicExpr::AO__c11_atomic_fetch_and:
915 case AtomicExpr::AO__atomic_fetch_and:
916 LibCallName = "__atomic_fetch_and";
917 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
918 MemTy, E->getExprLoc(), sizeChars);
920 // T __atomic_or_fetch_N(T *mem, T val, int order)
921 // T __atomic_fetch_or_N(T *mem, T val, int order)
922 case AtomicExpr::AO__atomic_or_fetch:
923 PostOp = llvm::Instruction::Or;
925 case AtomicExpr::AO__c11_atomic_fetch_or:
926 case AtomicExpr::AO__atomic_fetch_or:
927 LibCallName = "__atomic_fetch_or";
928 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
929 MemTy, E->getExprLoc(), sizeChars);
931 // T __atomic_sub_fetch_N(T *mem, T val, int order)
932 // T __atomic_fetch_sub_N(T *mem, T val, int order)
933 case AtomicExpr::AO__atomic_sub_fetch:
934 PostOp = llvm::Instruction::Sub;
936 case AtomicExpr::AO__c11_atomic_fetch_sub:
937 case AtomicExpr::AO__atomic_fetch_sub:
938 LibCallName = "__atomic_fetch_sub";
939 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
940 LoweredMemTy, E->getExprLoc(), sizeChars);
942 // T __atomic_xor_fetch_N(T *mem, T val, int order)
943 // T __atomic_fetch_xor_N(T *mem, T val, int order)
944 case AtomicExpr::AO__atomic_xor_fetch:
945 PostOp = llvm::Instruction::Xor;
947 case AtomicExpr::AO__c11_atomic_fetch_xor:
948 case AtomicExpr::AO__atomic_fetch_xor:
949 LibCallName = "__atomic_fetch_xor";
950 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
951 MemTy, E->getExprLoc(), sizeChars);
953 // T __atomic_nand_fetch_N(T *mem, T val, int order)
954 // T __atomic_fetch_nand_N(T *mem, T val, int order)
955 case AtomicExpr::AO__atomic_nand_fetch:
956 PostOp = llvm::Instruction::And; // the NOT is special cased below
958 case AtomicExpr::AO__atomic_fetch_nand:
959 LibCallName = "__atomic_fetch_nand";
960 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
961 MemTy, E->getExprLoc(), sizeChars);
965 // Optimized functions have the size in their name.
966 if (UseOptimizedLibcall)
967 LibCallName += "_" + llvm::utostr(Size);
968 // By default, assume we return a value of the atomic type.
970 if (UseOptimizedLibcall) {
971 // Value is returned directly.
972 // The function returns an appropriately sized integer type.
973 RetTy = getContext().getIntTypeForBitwidth(
974 getContext().toBits(sizeChars), /*Signed=*/false);
976 // Value is returned through parameter before the order.
977 RetTy = getContext().VoidTy;
978 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
979 getContext().VoidPtrTy);
982 // order is always the last parameter
983 Args.add(RValue::get(Order),
986 // PostOp is only needed for the atomic_*_fetch operations, and
987 // thus is only needed for and implemented in the
988 // UseOptimizedLibcall codepath.
989 assert(UseOptimizedLibcall || !PostOp);
991 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
992 // The value is returned directly from the libcall.
996 // The value is returned directly for optimized libcalls but the expr
997 // provided an out-param.
998 if (UseOptimizedLibcall && Res.getScalarVal()) {
999 llvm::Value *ResVal = Res.getScalarVal();
1001 llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1002 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1004 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1005 ResVal = Builder.CreateNot(ResVal);
1007 Builder.CreateStore(
1009 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1012 if (RValTy->isVoidType())
1013 return RValue::get(nullptr);
1015 return convertTempToRValue(
1016 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1017 RValTy, E->getExprLoc());
1020 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1021 E->getOp() == AtomicExpr::AO__atomic_store ||
1022 E->getOp() == AtomicExpr::AO__atomic_store_n;
1023 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1024 E->getOp() == AtomicExpr::AO__atomic_load ||
1025 E->getOp() == AtomicExpr::AO__atomic_load_n;
1027 if (isa<llvm::ConstantInt>(Order)) {
1028 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1029 // We should not ever get to a case where the ordering isn't a valid C ABI
1030 // value, but it's hard to enforce that in general.
1031 if (llvm::isValidAtomicOrderingCABI(ord))
1032 switch ((llvm::AtomicOrderingCABI)ord) {
1033 case llvm::AtomicOrderingCABI::relaxed:
1034 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1035 llvm::AtomicOrdering::Monotonic);
1037 case llvm::AtomicOrderingCABI::consume:
1038 case llvm::AtomicOrderingCABI::acquire:
1040 break; // Avoid crashing on code with undefined behavior
1041 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1042 llvm::AtomicOrdering::Acquire);
1044 case llvm::AtomicOrderingCABI::release:
1046 break; // Avoid crashing on code with undefined behavior
1047 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1048 llvm::AtomicOrdering::Release);
1050 case llvm::AtomicOrderingCABI::acq_rel:
1051 if (IsLoad || IsStore)
1052 break; // Avoid crashing on code with undefined behavior
1053 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1054 llvm::AtomicOrdering::AcquireRelease);
1056 case llvm::AtomicOrderingCABI::seq_cst:
1057 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1058 llvm::AtomicOrdering::SequentiallyConsistent);
1061 if (RValTy->isVoidType())
1062 return RValue::get(nullptr);
1064 return convertTempToRValue(
1065 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1066 RValTy, E->getExprLoc());
1069 // Long case, when Order isn't obviously constant.
1071 // Create all the relevant BB's
1072 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1073 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1074 *SeqCstBB = nullptr;
1075 MonotonicBB = createBasicBlock("monotonic", CurFn);
1077 AcquireBB = createBasicBlock("acquire", CurFn);
1079 ReleaseBB = createBasicBlock("release", CurFn);
1080 if (!IsLoad && !IsStore)
1081 AcqRelBB = createBasicBlock("acqrel", CurFn);
1082 SeqCstBB = createBasicBlock("seqcst", CurFn);
1083 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1085 // Create the switch for the split
1086 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1087 // doesn't matter unless someone is crazy enough to use something that
1088 // doesn't fold to a constant for the ordering.
1089 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1090 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1092 // Emit all the different atomics
1093 Builder.SetInsertPoint(MonotonicBB);
1094 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1095 Size, llvm::AtomicOrdering::Monotonic);
1096 Builder.CreateBr(ContBB);
1098 Builder.SetInsertPoint(AcquireBB);
1099 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1100 Size, llvm::AtomicOrdering::Acquire);
1101 Builder.CreateBr(ContBB);
1102 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1104 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1108 Builder.SetInsertPoint(ReleaseBB);
1109 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1110 Size, llvm::AtomicOrdering::Release);
1111 Builder.CreateBr(ContBB);
1112 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1115 if (!IsLoad && !IsStore) {
1116 Builder.SetInsertPoint(AcqRelBB);
1117 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1118 Size, llvm::AtomicOrdering::AcquireRelease);
1119 Builder.CreateBr(ContBB);
1120 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1123 Builder.SetInsertPoint(SeqCstBB);
1124 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1125 Size, llvm::AtomicOrdering::SequentiallyConsistent);
1126 Builder.CreateBr(ContBB);
1127 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1130 // Cleanup and return
1131 Builder.SetInsertPoint(ContBB);
1132 if (RValTy->isVoidType())
1133 return RValue::get(nullptr);
1135 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1136 return convertTempToRValue(
1137 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1138 RValTy, E->getExprLoc());
1141 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1142 unsigned addrspace =
1143 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1144 llvm::IntegerType *ty =
1145 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1146 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1149 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1150 llvm::Type *Ty = Addr.getElementType();
1151 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1152 if (SourceSizeInBits != AtomicSizeInBits) {
1153 Address Tmp = CreateTempAlloca();
1154 CGF.Builder.CreateMemCpy(Tmp, Addr,
1155 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1159 return emitCastToAtomicIntPointer(Addr);
1162 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1163 AggValueSlot resultSlot,
1165 bool asValue) const {
1166 if (LVal.isSimple()) {
1167 if (EvaluationKind == TEK_Aggregate)
1168 return resultSlot.asRValue();
1170 // Drill into the padding structure if we have one.
1172 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1174 // Otherwise, just convert the temporary to an r-value using the
1175 // normal conversion routine.
1176 return CGF.convertTempToRValue(addr, getValueType(), loc);
1179 // Get RValue from temp memory as atomic for non-simple lvalues
1180 return RValue::get(CGF.Builder.CreateLoad(addr));
1181 if (LVal.isBitField())
1182 return CGF.EmitLoadOfBitfieldLValue(
1183 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1184 LVal.getBaseInfo()), loc);
1185 if (LVal.isVectorElt())
1186 return CGF.EmitLoadOfLValue(
1187 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1188 LVal.getBaseInfo()), loc);
1189 assert(LVal.isExtVectorElt());
1190 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1191 addr, LVal.getExtVectorElts(), LVal.getType(),
1192 LVal.getBaseInfo()));
1195 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1196 AggValueSlot ResultSlot,
1198 bool AsValue) const {
1199 // Try not to in some easy cases.
1200 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1201 if (getEvaluationKind() == TEK_Scalar &&
1202 (((!LVal.isBitField() ||
1203 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1206 auto *ValTy = AsValue
1207 ? CGF.ConvertTypeForMem(ValueTy)
1208 : getAtomicAddress().getType()->getPointerElementType();
1209 if (ValTy->isIntegerTy()) {
1210 assert(IntVal->getType() == ValTy && "Different integer types.");
1211 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1212 } else if (ValTy->isPointerTy())
1213 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1214 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1215 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1218 // Create a temporary. This needs to be big enough to hold the
1220 Address Temp = Address::invalid();
1221 bool TempIsVolatile = false;
1222 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1223 assert(!ResultSlot.isIgnored());
1224 Temp = ResultSlot.getAddress();
1225 TempIsVolatile = ResultSlot.isVolatile();
1227 Temp = CreateTempAlloca();
1230 // Slam the integer into the temporary.
1231 Address CastTemp = emitCastToAtomicIntPointer(Temp);
1232 CGF.Builder.CreateStore(IntVal, CastTemp)
1233 ->setVolatile(TempIsVolatile);
1235 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1238 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1239 llvm::AtomicOrdering AO, bool) {
1240 // void __atomic_load(size_t size, void *mem, void *return, int order);
1242 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1243 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1244 CGF.getContext().VoidPtrTy);
1245 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1246 CGF.getContext().VoidPtrTy);
1248 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1249 CGF.getContext().IntTy);
1250 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1253 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1255 // Okay, we're doing this natively.
1256 Address Addr = getAtomicAddressAsAtomicIntPointer();
1257 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1258 Load->setAtomic(AO);
1260 // Other decoration.
1262 Load->setVolatile(true);
1263 if (LVal.getTBAAInfo())
1264 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1268 /// An LValue is a candidate for having its loads and stores be made atomic if
1269 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1270 /// performing such an operation can be performed without a libcall.
1271 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1272 if (!CGM.getCodeGenOpts().MSVolatile) return false;
1273 AtomicInfo AI(*this, LV);
1274 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1275 // An atomic is inline if we don't need to use a libcall.
1276 bool AtomicIsInline = !AI.shouldUseLibcall();
1277 // MSVC doesn't seem to do this for types wider than a pointer.
1278 if (getContext().getTypeSize(LV.getType()) >
1279 getContext().getTypeSize(getContext().getIntPtrType()))
1281 return IsVolatile && AtomicIsInline;
1284 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1285 AggValueSlot Slot) {
1286 llvm::AtomicOrdering AO;
1287 bool IsVolatile = LV.isVolatileQualified();
1288 if (LV.getType()->isAtomicType()) {
1289 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1291 AO = llvm::AtomicOrdering::Acquire;
1294 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1297 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1298 bool AsValue, llvm::AtomicOrdering AO,
1300 // Check whether we should use a library call.
1301 if (shouldUseLibcall()) {
1302 Address TempAddr = Address::invalid();
1303 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1304 assert(getEvaluationKind() == TEK_Aggregate);
1305 TempAddr = ResultSlot.getAddress();
1307 TempAddr = CreateTempAlloca();
1309 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1311 // Okay, turn that back into the original value or whole atomic (for
1312 // non-simple lvalues) type.
1313 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1316 // Okay, we're doing this natively.
1317 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1319 // If we're ignoring an aggregate return, don't do anything.
1320 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1321 return RValue::getAggregate(Address::invalid(), false);
1323 // Okay, turn that back into the original value or atomic (for non-simple
1325 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1328 /// Emit a load from an l-value of atomic type. Note that the r-value
1329 /// we produce is an r-value of the atomic *value* type.
1330 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1331 llvm::AtomicOrdering AO, bool IsVolatile,
1332 AggValueSlot resultSlot) {
1333 AtomicInfo Atomics(*this, src);
1334 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1338 /// Copy an r-value into memory as part of storing to an atomic type.
1339 /// This needs to create a bit-pattern suitable for atomic operations.
1340 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1341 assert(LVal.isSimple());
1342 // If we have an r-value, the rvalue should be of the atomic type,
1343 // which means that the caller is responsible for having zeroed
1344 // any padding. Just do an aggregate copy of that type.
1345 if (rvalue.isAggregate()) {
1346 CGF.EmitAggregateCopy(getAtomicAddress(),
1347 rvalue.getAggregateAddress(),
1349 (rvalue.isVolatileQualified()
1350 || LVal.isVolatileQualified()));
1354 // Okay, otherwise we're copying stuff.
1356 // Zero out the buffer if necessary.
1357 emitMemSetZeroIfNecessary();
1359 // Drill past the padding if present.
1360 LValue TempLVal = projectValue();
1362 // Okay, store the rvalue in.
1363 if (rvalue.isScalar()) {
1364 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1366 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1371 /// Materialize an r-value into memory for the purposes of storing it
1372 /// to an atomic type.
1373 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1374 // Aggregate r-values are already in memory, and EmitAtomicStore
1375 // requires them to be values of the atomic type.
1376 if (rvalue.isAggregate())
1377 return rvalue.getAggregateAddress();
1379 // Otherwise, make a temporary and materialize into it.
1380 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1381 AtomicInfo Atomics(CGF, TempLV);
1382 Atomics.emitCopyIntoMemory(rvalue);
1383 return TempLV.getAddress();
1386 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1387 // If we've got a scalar value of the right size, try to avoid going
1389 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1390 llvm::Value *Value = RVal.getScalarVal();
1391 if (isa<llvm::IntegerType>(Value->getType()))
1392 return CGF.EmitToMemory(Value, ValueTy);
1394 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1395 CGF.getLLVMContext(),
1396 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1397 if (isa<llvm::PointerType>(Value->getType()))
1398 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1399 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1400 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1403 // Otherwise, we need to go through memory.
1404 // Put the r-value in memory.
1405 Address Addr = materializeRValue(RVal);
1407 // Cast the temporary to the atomic int type and pull a value out.
1408 Addr = emitCastToAtomicIntPointer(Addr);
1409 return CGF.Builder.CreateLoad(Addr);
1412 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1413 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1414 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1415 // Do the atomic store.
1416 Address Addr = getAtomicAddressAsAtomicIntPointer();
1417 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1418 ExpectedVal, DesiredVal,
1420 // Other decoration.
1421 Inst->setVolatile(LVal.isVolatileQualified());
1422 Inst->setWeak(IsWeak);
1424 // Okay, turn that back into the original value type.
1425 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1426 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1427 return std::make_pair(PreviousVal, SuccessFailureVal);
1431 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1432 llvm::Value *DesiredAddr,
1433 llvm::AtomicOrdering Success,
1434 llvm::AtomicOrdering Failure) {
1435 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1436 // void *desired, int success, int failure);
1438 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1439 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1440 CGF.getContext().VoidPtrTy);
1441 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1442 CGF.getContext().VoidPtrTy);
1443 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1444 CGF.getContext().VoidPtrTy);
1445 Args.add(RValue::get(
1446 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1447 CGF.getContext().IntTy);
1448 Args.add(RValue::get(
1449 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1450 CGF.getContext().IntTy);
1451 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1452 CGF.getContext().BoolTy, Args);
1454 return SuccessFailureRVal.getScalarVal();
1457 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1458 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1459 llvm::AtomicOrdering Failure, bool IsWeak) {
1460 if (isStrongerThan(Failure, Success))
1461 // Don't assert on undefined behavior "failure argument shall be no stronger
1462 // than the success argument".
1463 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1465 // Check whether we should use a library call.
1466 if (shouldUseLibcall()) {
1467 // Produce a source address.
1468 Address ExpectedAddr = materializeRValue(Expected);
1469 Address DesiredAddr = materializeRValue(Desired);
1470 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1471 DesiredAddr.getPointer(),
1473 return std::make_pair(
1474 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1475 SourceLocation(), /*AsValue=*/false),
1479 // If we've got a scalar value of the right size, try to avoid going
1481 auto *ExpectedVal = convertRValueToInt(Expected);
1482 auto *DesiredVal = convertRValueToInt(Desired);
1483 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1485 return std::make_pair(
1486 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1487 SourceLocation(), /*AsValue=*/false),
1492 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1493 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1494 Address DesiredAddr) {
1496 LValue AtomicLVal = Atomics.getAtomicLValue();
1498 if (AtomicLVal.isSimple()) {
1500 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1502 // Build new lvalue for temp address
1503 Address Ptr = Atomics.materializeRValue(OldRVal);
1505 if (AtomicLVal.isBitField()) {
1507 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1508 AtomicLVal.getType(),
1509 AtomicLVal.getBaseInfo());
1511 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1512 AtomicLVal.getType(),
1513 AtomicLVal.getBaseInfo());
1514 } else if (AtomicLVal.isVectorElt()) {
1515 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1516 AtomicLVal.getType(),
1517 AtomicLVal.getBaseInfo());
1518 DesiredLVal = LValue::MakeVectorElt(
1519 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1520 AtomicLVal.getBaseInfo());
1522 assert(AtomicLVal.isExtVectorElt());
1523 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1524 AtomicLVal.getType(),
1525 AtomicLVal.getBaseInfo());
1526 DesiredLVal = LValue::MakeExtVectorElt(
1527 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1528 AtomicLVal.getBaseInfo());
1530 UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1531 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1532 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1534 // Store new value in the corresponding memory area
1535 RValue NewRVal = UpdateOp(UpRVal);
1536 if (NewRVal.isScalar()) {
1537 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1539 assert(NewRVal.isComplex());
1540 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1545 void AtomicInfo::EmitAtomicUpdateLibcall(
1546 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1548 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1550 Address ExpectedAddr = CreateTempAlloca();
1552 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1553 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1554 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1555 CGF.EmitBlock(ContBB);
1556 Address DesiredAddr = CreateTempAlloca();
1557 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1558 requiresMemSetZero(getAtomicAddress().getElementType())) {
1559 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1560 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1562 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1563 AggValueSlot::ignored(),
1564 SourceLocation(), /*AsValue=*/false);
1565 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1567 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1568 DesiredAddr.getPointer(),
1570 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1571 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1574 void AtomicInfo::EmitAtomicUpdateOp(
1575 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1577 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1579 // Do the atomic load.
1580 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1581 // For non-simple lvalues perform compare-and-swap procedure.
1582 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1583 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1584 auto *CurBB = CGF.Builder.GetInsertBlock();
1585 CGF.EmitBlock(ContBB);
1586 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1587 /*NumReservedValues=*/2);
1588 PHI->addIncoming(OldVal, CurBB);
1589 Address NewAtomicAddr = CreateTempAlloca();
1590 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1591 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1592 requiresMemSetZero(getAtomicAddress().getElementType())) {
1593 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1595 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1596 SourceLocation(), /*AsValue=*/false);
1597 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1598 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1599 // Try to write new value using cmpxchg operation
1600 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1601 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1602 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1603 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1607 RValue UpdateRVal, Address DesiredAddr) {
1608 LValue AtomicLVal = Atomics.getAtomicLValue();
1610 // Build new lvalue for temp address
1611 if (AtomicLVal.isBitField()) {
1613 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1614 AtomicLVal.getType(),
1615 AtomicLVal.getBaseInfo());
1616 } else if (AtomicLVal.isVectorElt()) {
1618 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1619 AtomicLVal.getType(),
1620 AtomicLVal.getBaseInfo());
1622 assert(AtomicLVal.isExtVectorElt());
1623 DesiredLVal = LValue::MakeExtVectorElt(
1624 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1625 AtomicLVal.getBaseInfo());
1627 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1628 // Store new value in the corresponding memory area
1629 assert(UpdateRVal.isScalar());
1630 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1633 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1634 RValue UpdateRVal, bool IsVolatile) {
1635 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1637 Address ExpectedAddr = CreateTempAlloca();
1639 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1640 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1641 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1642 CGF.EmitBlock(ContBB);
1643 Address DesiredAddr = CreateTempAlloca();
1644 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1645 requiresMemSetZero(getAtomicAddress().getElementType())) {
1646 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1647 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1649 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1651 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1652 DesiredAddr.getPointer(),
1654 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1655 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1658 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1660 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1662 // Do the atomic load.
1663 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1664 // For non-simple lvalues perform compare-and-swap procedure.
1665 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1666 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1667 auto *CurBB = CGF.Builder.GetInsertBlock();
1668 CGF.EmitBlock(ContBB);
1669 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1670 /*NumReservedValues=*/2);
1671 PHI->addIncoming(OldVal, CurBB);
1672 Address NewAtomicAddr = CreateTempAlloca();
1673 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1674 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1675 requiresMemSetZero(getAtomicAddress().getElementType())) {
1676 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1678 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1679 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1680 // Try to write new value using cmpxchg operation
1681 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1682 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1683 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1684 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1687 void AtomicInfo::EmitAtomicUpdate(
1688 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1690 if (shouldUseLibcall()) {
1691 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1693 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1697 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1699 if (shouldUseLibcall()) {
1700 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1702 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1706 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1708 bool IsVolatile = lvalue.isVolatileQualified();
1709 llvm::AtomicOrdering AO;
1710 if (lvalue.getType()->isAtomicType()) {
1711 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1713 AO = llvm::AtomicOrdering::Release;
1716 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1719 /// Emit a store to an l-value of atomic type.
1721 /// Note that the r-value is expected to be an r-value *of the atomic
1722 /// type*; this means that for aggregate r-values, it should include
1723 /// storage for any padding that was necessary.
1724 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1725 llvm::AtomicOrdering AO, bool IsVolatile,
1727 // If this is an aggregate r-value, it should agree in type except
1728 // maybe for address-space qualification.
1729 assert(!rvalue.isAggregate() ||
1730 rvalue.getAggregateAddress().getElementType()
1731 == dest.getAddress().getElementType());
1733 AtomicInfo atomics(*this, dest);
1734 LValue LVal = atomics.getAtomicLValue();
1736 // If this is an initialization, just put the value there normally.
1737 if (LVal.isSimple()) {
1739 atomics.emitCopyIntoMemory(rvalue);
1743 // Check whether we should use a library call.
1744 if (atomics.shouldUseLibcall()) {
1745 // Produce a source address.
1746 Address srcAddr = atomics.materializeRValue(rvalue);
1748 // void __atomic_store(size_t size, void *mem, void *val, int order)
1750 args.add(RValue::get(atomics.getAtomicSizeValue()),
1751 getContext().getSizeType());
1752 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1753 getContext().VoidPtrTy);
1754 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1755 getContext().VoidPtrTy);
1757 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1758 getContext().IntTy);
1759 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1763 // Okay, we're doing this natively.
1764 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1766 // Do the atomic store.
1768 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1769 intValue = Builder.CreateIntCast(
1770 intValue, addr.getElementType(), /*isSigned=*/false);
1771 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1773 // Initializations don't need to be atomic.
1775 store->setAtomic(AO);
1777 // Other decoration.
1779 store->setVolatile(true);
1780 if (dest.getTBAAInfo())
1781 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1785 // Emit simple atomic update operation.
1786 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1789 /// Emit a compare-and-exchange op for atomic type.
1791 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1792 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1793 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1794 AggValueSlot Slot) {
1795 // If this is an aggregate r-value, it should agree in type except
1796 // maybe for address-space qualification.
1797 assert(!Expected.isAggregate() ||
1798 Expected.getAggregateAddress().getElementType() ==
1799 Obj.getAddress().getElementType());
1800 assert(!Desired.isAggregate() ||
1801 Desired.getAggregateAddress().getElementType() ==
1802 Obj.getAddress().getElementType());
1803 AtomicInfo Atomics(*this, Obj);
1805 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1809 void CodeGenFunction::EmitAtomicUpdate(
1810 LValue LVal, llvm::AtomicOrdering AO,
1811 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1812 AtomicInfo Atomics(*this, LVal);
1813 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1816 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1817 AtomicInfo atomics(*this, dest);
1819 switch (atomics.getEvaluationKind()) {
1821 llvm::Value *value = EmitScalarExpr(init);
1822 atomics.emitCopyIntoMemory(RValue::get(value));
1827 ComplexPairTy value = EmitComplexExpr(init);
1828 atomics.emitCopyIntoMemory(RValue::getComplex(value));
1832 case TEK_Aggregate: {
1833 // Fix up the destination if the initializer isn't an expression
1835 bool Zeroed = false;
1836 if (!init->getType()->isAtomicType()) {
1837 Zeroed = atomics.emitMemSetZeroIfNecessary();
1838 dest = atomics.projectValue();
1841 // Evaluate the expression directly into the destination.
1842 AggValueSlot slot = AggValueSlot::forLValue(dest,
1843 AggValueSlot::IsNotDestructed,
1844 AggValueSlot::DoesNotNeedGCBarriers,
1845 AggValueSlot::IsNotAliased,
1846 Zeroed ? AggValueSlot::IsZeroed :
1847 AggValueSlot::IsNotZeroed);
1849 EmitAggExpr(init, slot);
1853 llvm_unreachable("bad evaluation kind");