1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the code for emitting atomic operations.
12 //===----------------------------------------------------------------------===//
14 #include "CodeGenFunction.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Operator.h"
25 using namespace clang;
26 using namespace CodeGen;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
37 CharUnits LValueAlign;
38 TypeEvaluationKind EvaluationKind;
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .RoundUpToAlignment(lvalue.getAlignment()));
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88 VoidPtrAddr, OffsetInChars.getQuantity());
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92 "atomic_bitfield_base");
95 BFI.StorageSize = AtomicSizeInBits;
96 BFI.StorageOffset += OffsetInChars;
97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98 BFI, lvalue.getType(),
99 lvalue.getAlignmentSource());
100 LVal.setTBAAInfo(lvalue.getTBAAInfo());
101 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102 if (AtomicTy.isNull()) {
105 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107 /*IndexTypeQuals=*/0);
109 AtomicAlign = ValueAlign = lvalue.getAlignment();
110 } else if (lvalue.isVectorElt()) {
111 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112 ValueSizeInBits = C.getTypeSize(ValueTy);
113 AtomicTy = lvalue.getType();
114 AtomicSizeInBits = C.getTypeSize(AtomicTy);
115 AtomicAlign = ValueAlign = lvalue.getAlignment();
118 assert(lvalue.isExtVectorElt());
119 ValueTy = lvalue.getType();
120 ValueSizeInBits = C.getTypeSize(ValueTy);
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122 lvalue.getType(), lvalue.getExtVectorAddress()
123 .getElementType()->getVectorNumElements());
124 AtomicSizeInBits = C.getTypeSize(AtomicTy);
125 AtomicAlign = ValueAlign = lvalue.getAlignment();
128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
132 QualType getAtomicType() const { return AtomicTy; }
133 QualType getValueType() const { return ValueTy; }
134 CharUnits getAtomicAlignment() const { return AtomicAlign; }
135 CharUnits getValueAlignment() const { return ValueAlign; }
136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139 bool shouldUseLibcall() const { return UseLibcall; }
140 const LValue &getAtomicLValue() const { return LVal; }
141 llvm::Value *getAtomicPointer() const {
143 return LVal.getPointer();
144 else if (LVal.isBitField())
145 return LVal.getBitFieldPointer();
146 else if (LVal.isVectorElt())
147 return LVal.getVectorPointer();
148 assert(LVal.isExtVectorElt());
149 return LVal.getExtVectorPointer();
151 Address getAtomicAddress() const {
152 return Address(getAtomicPointer(), getAtomicAlignment());
155 Address getAtomicAddressAsAtomicIntPointer() const {
156 return emitCastToAtomicIntPointer(getAtomicAddress());
159 /// Is the atomic size larger than the underlying value type?
161 /// Note that the absence of padding does not mean that atomic
162 /// objects are completely interchangeable with non-atomic
163 /// objects: we might have promoted the alignment of a type
164 /// without making it bigger.
165 bool hasPadding() const {
166 return (ValueSizeInBits != AtomicSizeInBits);
169 bool emitMemSetZeroIfNecessary() const;
171 llvm::Value *getAtomicSizeValue() const {
172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173 return CGF.CGM.getSize(size);
176 /// Cast the given pointer to an integer pointer suitable for atomic
177 /// operations if the source.
178 Address emitCastToAtomicIntPointer(Address Addr) const;
180 /// If Addr is compatible with the iN that will be used for an atomic
181 /// operation, bitcast it. Otherwise, create a temporary that is suitable
182 /// and copy the value across.
183 Address convertToAtomicIntPointer(Address Addr) const;
185 /// Turn an atomic-layout object into an r-value.
186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187 SourceLocation loc, bool AsValue) const;
189 /// \brief Converts a rvalue to integer value.
190 llvm::Value *convertRValueToInt(RValue RVal) const;
192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193 AggValueSlot ResultSlot,
194 SourceLocation Loc, bool AsValue) const;
196 /// Copy an atomic r-value into atomic-layout memory.
197 void emitCopyIntoMemory(RValue rvalue) const;
199 /// Project an l-value down to the value field.
200 LValue projectValue() const {
201 assert(LVal.isSimple());
202 Address addr = getAtomicAddress();
204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207 LVal.getAlignmentSource(), LVal.getTBAAInfo());
210 /// \brief Emits atomic load.
211 /// \returns Loaded value.
212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213 bool AsValue, llvm::AtomicOrdering AO,
216 /// \brief Emits atomic compare-and-exchange sequence.
217 /// \param Expected Expected value.
218 /// \param Desired Desired value.
219 /// \param Success Atomic ordering for success operation.
220 /// \param Failure Atomic ordering for failed operation.
221 /// \param IsWeak true if atomic operation is weak, false otherwise.
222 /// \returns Pair of values: previous value from storage (value type) and
223 /// boolean flag (i1 type) with true if success and false otherwise.
224 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
225 RValue Expected, RValue Desired,
226 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
227 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
228 bool IsWeak = false);
230 /// \brief Emits atomic update.
231 /// \param AO Atomic ordering.
232 /// \param UpdateOp Update operation for the current lvalue.
233 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
234 const llvm::function_ref<RValue(RValue)> &UpdateOp,
236 /// \brief Emits atomic update.
237 /// \param AO Atomic ordering.
238 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241 /// Materialize an atomic r-value in atomic-layout memory.
242 Address materializeRValue(RValue rvalue) const;
244 /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
246 static AtomicExpr::AtomicOrderingKind
247 translateAtomicOrdering(const llvm::AtomicOrdering AO);
249 /// \brief Creates temp alloca for intermediate operations on atomic value.
250 Address CreateTempAlloca() const;
252 bool requiresMemSetZero(llvm::Type *type) const;
255 /// \brief Emits atomic load as a libcall.
256 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
257 llvm::AtomicOrdering AO, bool IsVolatile);
258 /// \brief Emits atomic load as LLVM instruction.
259 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
260 /// \brief Emits atomic compare-and-exchange op as a libcall.
261 llvm::Value *EmitAtomicCompareExchangeLibcall(
262 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
263 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
264 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
265 /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
266 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
267 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
268 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
269 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
270 bool IsWeak = false);
271 /// \brief Emit atomic update as libcalls.
273 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
274 const llvm::function_ref<RValue(RValue)> &UpdateOp,
276 /// \brief Emit atomic update as LLVM instructions.
277 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
278 const llvm::function_ref<RValue(RValue)> &UpdateOp,
280 /// \brief Emit atomic update as libcalls.
281 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283 /// \brief Emit atomic update as LLVM instructions.
284 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
289 AtomicExpr::AtomicOrderingKind
290 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
292 case llvm::Unordered:
293 case llvm::NotAtomic:
294 case llvm::Monotonic:
295 return AtomicExpr::AO_ABI_memory_order_relaxed;
297 return AtomicExpr::AO_ABI_memory_order_acquire;
299 return AtomicExpr::AO_ABI_memory_order_release;
300 case llvm::AcquireRelease:
301 return AtomicExpr::AO_ABI_memory_order_acq_rel;
302 case llvm::SequentiallyConsistent:
303 return AtomicExpr::AO_ABI_memory_order_seq_cst;
305 llvm_unreachable("Unhandled AtomicOrdering");
308 Address AtomicInfo::CreateTempAlloca() const {
309 Address TempAlloca = CGF.CreateMemTemp(
310 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
312 getAtomicAlignment(),
314 // Cast to pointer to value type for bitfields.
315 if (LVal.isBitField())
316 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
317 TempAlloca, getAtomicAddress().getType());
321 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
325 const CGFunctionInfo &fnInfo =
326 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
327 FunctionType::ExtInfo(), RequiredArgs::All);
328 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
329 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
330 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
333 /// Does a store of the given IR type modify the full expected width?
334 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
335 uint64_t expectedSize) {
336 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
339 /// Does the atomic type require memsetting to zero before initialization?
341 /// The IR type is provided as a way of making certain queries faster.
342 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
343 // If the atomic type has size padding, we definitely need a memset.
344 if (hasPadding()) return true;
346 // Otherwise, do some simple heuristics to try to avoid it:
347 switch (getEvaluationKind()) {
348 // For scalars and complexes, check whether the store size of the
349 // type uses the full size.
351 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
353 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
354 AtomicSizeInBits / 2);
356 // Padding in structs has an undefined bit pattern. User beware.
360 llvm_unreachable("bad evaluation kind");
363 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
364 assert(LVal.isSimple());
365 llvm::Value *addr = LVal.getPointer();
366 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
369 CGF.Builder.CreateMemSet(
370 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
371 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
372 LVal.getAlignment().getQuantity());
376 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377 Address Dest, Address Ptr,
378 Address Val1, Address Val2,
380 llvm::AtomicOrdering SuccessOrder,
381 llvm::AtomicOrdering FailureOrder) {
382 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
383 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
384 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
386 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
387 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
388 Pair->setVolatile(E->isVolatile());
389 Pair->setWeak(IsWeak);
391 // Cmp holds the result of the compare-exchange operation: true on success,
393 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
396 // This basic block is used to hold the store instruction if the operation
398 llvm::BasicBlock *StoreExpectedBB =
399 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
401 // This basic block is the exit point of the operation, we should end up
402 // here regardless of whether or not the operation succeeded.
403 llvm::BasicBlock *ContinueBB =
404 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
406 // Update Expected if Expected isn't equal to Old, otherwise branch to the
408 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
410 CGF.Builder.SetInsertPoint(StoreExpectedBB);
411 // Update the memory at Expected with Old's value.
412 CGF.Builder.CreateStore(Old, Val1);
413 // Finally, branch to the exit point.
414 CGF.Builder.CreateBr(ContinueBB);
416 CGF.Builder.SetInsertPoint(ContinueBB);
417 // Update the memory at Dest with Cmp's value.
418 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
421 /// Given an ordering required on success, emit all possible cmpxchg
422 /// instructions to cope with the provided (but possibly only dynamically known)
424 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
425 bool IsWeak, Address Dest,
426 Address Ptr, Address Val1,
428 llvm::Value *FailureOrderVal,
430 llvm::AtomicOrdering SuccessOrder) {
431 llvm::AtomicOrdering FailureOrder;
432 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433 switch (FO->getSExtValue()) {
435 FailureOrder = llvm::Monotonic;
437 case AtomicExpr::AO_ABI_memory_order_consume:
438 case AtomicExpr::AO_ABI_memory_order_acquire:
439 FailureOrder = llvm::Acquire;
441 case AtomicExpr::AO_ABI_memory_order_seq_cst:
442 FailureOrder = llvm::SequentiallyConsistent;
445 if (FailureOrder >= SuccessOrder) {
446 // Don't assert on undefined behaviour.
448 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
450 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size,
451 SuccessOrder, FailureOrder);
455 // Create all the relevant BB's
456 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
458 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
459 if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
460 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
461 if (SuccessOrder == llvm::SequentiallyConsistent)
462 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
464 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
466 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
468 // Emit all the different atomics
470 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
471 // doesn't matter unless someone is crazy enough to use something that
472 // doesn't fold to a constant for the ordering.
473 CGF.Builder.SetInsertPoint(MonotonicBB);
474 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
475 Size, SuccessOrder, llvm::Monotonic);
476 CGF.Builder.CreateBr(ContBB);
479 CGF.Builder.SetInsertPoint(AcquireBB);
480 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
481 Size, SuccessOrder, llvm::Acquire);
482 CGF.Builder.CreateBr(ContBB);
483 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
485 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
489 CGF.Builder.SetInsertPoint(SeqCstBB);
490 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
491 Size, SuccessOrder, llvm::SequentiallyConsistent);
492 CGF.Builder.CreateBr(ContBB);
493 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
497 CGF.Builder.SetInsertPoint(ContBB);
500 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
501 Address Ptr, Address Val1, Address Val2,
502 llvm::Value *IsWeak, llvm::Value *FailureOrder,
503 uint64_t Size, llvm::AtomicOrdering Order) {
504 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
505 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
507 switch (E->getOp()) {
508 case AtomicExpr::AO__c11_atomic_init:
509 llvm_unreachable("Already handled!");
511 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
512 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
513 FailureOrder, Size, Order);
515 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
516 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
517 FailureOrder, Size, Order);
519 case AtomicExpr::AO__atomic_compare_exchange:
520 case AtomicExpr::AO__atomic_compare_exchange_n: {
521 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
522 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
523 Val1, Val2, FailureOrder, Size, Order);
525 // Create all the relevant BB's
526 llvm::BasicBlock *StrongBB =
527 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
528 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
529 llvm::BasicBlock *ContBB =
530 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
532 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
533 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
535 CGF.Builder.SetInsertPoint(StrongBB);
536 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
537 FailureOrder, Size, Order);
538 CGF.Builder.CreateBr(ContBB);
540 CGF.Builder.SetInsertPoint(WeakBB);
541 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
542 FailureOrder, Size, Order);
543 CGF.Builder.CreateBr(ContBB);
545 CGF.Builder.SetInsertPoint(ContBB);
549 case AtomicExpr::AO__c11_atomic_load:
550 case AtomicExpr::AO__atomic_load_n:
551 case AtomicExpr::AO__atomic_load: {
552 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
553 Load->setAtomic(Order);
554 Load->setVolatile(E->isVolatile());
555 CGF.Builder.CreateStore(Load, Dest);
559 case AtomicExpr::AO__c11_atomic_store:
560 case AtomicExpr::AO__atomic_store:
561 case AtomicExpr::AO__atomic_store_n: {
562 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
563 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
564 Store->setAtomic(Order);
565 Store->setVolatile(E->isVolatile());
569 case AtomicExpr::AO__c11_atomic_exchange:
570 case AtomicExpr::AO__atomic_exchange_n:
571 case AtomicExpr::AO__atomic_exchange:
572 Op = llvm::AtomicRMWInst::Xchg;
575 case AtomicExpr::AO__atomic_add_fetch:
576 PostOp = llvm::Instruction::Add;
578 case AtomicExpr::AO__c11_atomic_fetch_add:
579 case AtomicExpr::AO__atomic_fetch_add:
580 Op = llvm::AtomicRMWInst::Add;
583 case AtomicExpr::AO__atomic_sub_fetch:
584 PostOp = llvm::Instruction::Sub;
586 case AtomicExpr::AO__c11_atomic_fetch_sub:
587 case AtomicExpr::AO__atomic_fetch_sub:
588 Op = llvm::AtomicRMWInst::Sub;
591 case AtomicExpr::AO__atomic_and_fetch:
592 PostOp = llvm::Instruction::And;
594 case AtomicExpr::AO__c11_atomic_fetch_and:
595 case AtomicExpr::AO__atomic_fetch_and:
596 Op = llvm::AtomicRMWInst::And;
599 case AtomicExpr::AO__atomic_or_fetch:
600 PostOp = llvm::Instruction::Or;
602 case AtomicExpr::AO__c11_atomic_fetch_or:
603 case AtomicExpr::AO__atomic_fetch_or:
604 Op = llvm::AtomicRMWInst::Or;
607 case AtomicExpr::AO__atomic_xor_fetch:
608 PostOp = llvm::Instruction::Xor;
610 case AtomicExpr::AO__c11_atomic_fetch_xor:
611 case AtomicExpr::AO__atomic_fetch_xor:
612 Op = llvm::AtomicRMWInst::Xor;
615 case AtomicExpr::AO__atomic_nand_fetch:
616 PostOp = llvm::Instruction::And; // the NOT is special cased below
618 case AtomicExpr::AO__atomic_fetch_nand:
619 Op = llvm::AtomicRMWInst::Nand;
623 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
624 llvm::AtomicRMWInst *RMWI =
625 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
626 RMWI->setVolatile(E->isVolatile());
628 // For __atomic_*_fetch operations, perform the operation again to
629 // determine the value which was written.
630 llvm::Value *Result = RMWI;
632 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
633 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
634 Result = CGF.Builder.CreateNot(Result);
635 CGF.Builder.CreateStore(Result, Dest);
638 // This function emits any expression (scalar, complex, or aggregate)
639 // into a temporary alloca.
641 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
642 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
643 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
649 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
650 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
651 SourceLocation Loc, CharUnits SizeInChars) {
652 if (UseOptimizedLibcall) {
653 // Load value and pass it to the function directly.
654 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
655 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
657 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
658 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
659 SizeInBits)->getPointerTo();
660 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
661 Val = CGF.EmitLoadOfScalar(Ptr, false,
662 CGF.getContext().getPointerType(ValTy),
664 // Coerce the value into an appropriately sized integer type.
665 Args.add(RValue::get(Val), ValTy);
667 // Non-optimized functions always take a reference.
668 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
669 CGF.getContext().VoidPtrTy);
673 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
674 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
675 QualType MemTy = AtomicTy;
676 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
677 MemTy = AT->getValueType();
678 CharUnits sizeChars, alignChars;
679 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
680 uint64_t Size = sizeChars.getQuantity();
681 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
682 bool UseLibcall = (sizeChars != alignChars ||
683 getContext().toBits(sizeChars) > MaxInlineWidthInBits);
685 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
687 Address Val1 = Address::invalid();
688 Address Val2 = Address::invalid();
689 Address Dest = Address::invalid();
690 Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
692 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
693 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
694 EmitAtomicInit(E->getVal1(), lvalue);
695 return RValue::get(nullptr);
698 llvm::Value *Order = EmitScalarExpr(E->getOrder());
700 switch (E->getOp()) {
701 case AtomicExpr::AO__c11_atomic_init:
702 llvm_unreachable("Already handled above with EmitAtomicInit!");
704 case AtomicExpr::AO__c11_atomic_load:
705 case AtomicExpr::AO__atomic_load_n:
708 case AtomicExpr::AO__atomic_load:
709 Dest = EmitPointerWithAlignment(E->getVal1());
712 case AtomicExpr::AO__atomic_store:
713 Val1 = EmitPointerWithAlignment(E->getVal1());
716 case AtomicExpr::AO__atomic_exchange:
717 Val1 = EmitPointerWithAlignment(E->getVal1());
718 Dest = EmitPointerWithAlignment(E->getVal2());
721 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
722 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
723 case AtomicExpr::AO__atomic_compare_exchange_n:
724 case AtomicExpr::AO__atomic_compare_exchange:
725 Val1 = EmitPointerWithAlignment(E->getVal1());
726 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
727 Val2 = EmitPointerWithAlignment(E->getVal2());
729 Val2 = EmitValToTemp(*this, E->getVal2());
730 OrderFail = EmitScalarExpr(E->getOrderFail());
731 if (E->getNumSubExprs() == 6)
732 IsWeak = EmitScalarExpr(E->getWeak());
735 case AtomicExpr::AO__c11_atomic_fetch_add:
736 case AtomicExpr::AO__c11_atomic_fetch_sub:
737 if (MemTy->isPointerType()) {
738 // For pointer arithmetic, we're required to do a bit of math:
739 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
740 // ... but only for the C11 builtins. The GNU builtins expect the
741 // user to multiply by sizeof(T).
742 QualType Val1Ty = E->getVal1()->getType();
743 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
744 CharUnits PointeeIncAmt =
745 getContext().getTypeSizeInChars(MemTy->getPointeeType());
746 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
747 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
749 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
753 case AtomicExpr::AO__atomic_fetch_add:
754 case AtomicExpr::AO__atomic_fetch_sub:
755 case AtomicExpr::AO__atomic_add_fetch:
756 case AtomicExpr::AO__atomic_sub_fetch:
757 case AtomicExpr::AO__c11_atomic_store:
758 case AtomicExpr::AO__c11_atomic_exchange:
759 case AtomicExpr::AO__atomic_store_n:
760 case AtomicExpr::AO__atomic_exchange_n:
761 case AtomicExpr::AO__c11_atomic_fetch_and:
762 case AtomicExpr::AO__c11_atomic_fetch_or:
763 case AtomicExpr::AO__c11_atomic_fetch_xor:
764 case AtomicExpr::AO__atomic_fetch_and:
765 case AtomicExpr::AO__atomic_fetch_or:
766 case AtomicExpr::AO__atomic_fetch_xor:
767 case AtomicExpr::AO__atomic_fetch_nand:
768 case AtomicExpr::AO__atomic_and_fetch:
769 case AtomicExpr::AO__atomic_or_fetch:
770 case AtomicExpr::AO__atomic_xor_fetch:
771 case AtomicExpr::AO__atomic_nand_fetch:
772 Val1 = EmitValToTemp(*this, E->getVal1());
776 QualType RValTy = E->getType().getUnqualifiedType();
778 // The inlined atomics only function on iN types, where N is a power of 2. We
779 // need to make sure (via temporaries if necessary) that all incoming values
781 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
782 AtomicInfo Atomics(*this, AtomicVal);
784 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
785 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
786 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
788 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
789 else if (E->isCmpXChg())
790 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
791 else if (!RValTy->isVoidType())
792 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
794 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
796 bool UseOptimizedLibcall = false;
797 switch (E->getOp()) {
798 case AtomicExpr::AO__c11_atomic_init:
799 llvm_unreachable("Already handled above with EmitAtomicInit!");
801 case AtomicExpr::AO__c11_atomic_fetch_add:
802 case AtomicExpr::AO__atomic_fetch_add:
803 case AtomicExpr::AO__c11_atomic_fetch_and:
804 case AtomicExpr::AO__atomic_fetch_and:
805 case AtomicExpr::AO__c11_atomic_fetch_or:
806 case AtomicExpr::AO__atomic_fetch_or:
807 case AtomicExpr::AO__atomic_fetch_nand:
808 case AtomicExpr::AO__c11_atomic_fetch_sub:
809 case AtomicExpr::AO__atomic_fetch_sub:
810 case AtomicExpr::AO__c11_atomic_fetch_xor:
811 case AtomicExpr::AO__atomic_fetch_xor:
812 case AtomicExpr::AO__atomic_add_fetch:
813 case AtomicExpr::AO__atomic_and_fetch:
814 case AtomicExpr::AO__atomic_nand_fetch:
815 case AtomicExpr::AO__atomic_or_fetch:
816 case AtomicExpr::AO__atomic_sub_fetch:
817 case AtomicExpr::AO__atomic_xor_fetch:
818 // For these, only library calls for certain sizes exist.
819 UseOptimizedLibcall = true;
822 case AtomicExpr::AO__c11_atomic_load:
823 case AtomicExpr::AO__c11_atomic_store:
824 case AtomicExpr::AO__c11_atomic_exchange:
825 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
826 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
827 case AtomicExpr::AO__atomic_load_n:
828 case AtomicExpr::AO__atomic_load:
829 case AtomicExpr::AO__atomic_store_n:
830 case AtomicExpr::AO__atomic_store:
831 case AtomicExpr::AO__atomic_exchange_n:
832 case AtomicExpr::AO__atomic_exchange:
833 case AtomicExpr::AO__atomic_compare_exchange_n:
834 case AtomicExpr::AO__atomic_compare_exchange:
835 // Only use optimized library calls for sizes for which they exist.
836 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
837 UseOptimizedLibcall = true;
842 if (!UseOptimizedLibcall) {
843 // For non-optimized library calls, the size is the first parameter
844 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
845 getContext().getSizeType());
847 // Atomic address is the first or second parameter
848 Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())),
849 getContext().VoidPtrTy);
851 std::string LibCallName;
852 QualType LoweredMemTy =
853 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
855 bool HaveRetTy = false;
856 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
857 switch (E->getOp()) {
858 case AtomicExpr::AO__c11_atomic_init:
859 llvm_unreachable("Already handled!");
861 // There is only one libcall for compare an exchange, because there is no
862 // optimisation benefit possible from a libcall version of a weak compare
864 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
865 // void *desired, int success, int failure)
866 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
867 // int success, int failure)
868 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
869 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
870 case AtomicExpr::AO__atomic_compare_exchange:
871 case AtomicExpr::AO__atomic_compare_exchange_n:
872 LibCallName = "__atomic_compare_exchange";
873 RetTy = getContext().BoolTy;
875 Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())),
876 getContext().VoidPtrTy);
877 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
878 MemTy, E->getExprLoc(), sizeChars);
879 Args.add(RValue::get(Order), getContext().IntTy);
882 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
884 // T __atomic_exchange_N(T *mem, T val, int order)
885 case AtomicExpr::AO__c11_atomic_exchange:
886 case AtomicExpr::AO__atomic_exchange_n:
887 case AtomicExpr::AO__atomic_exchange:
888 LibCallName = "__atomic_exchange";
889 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
890 MemTy, E->getExprLoc(), sizeChars);
892 // void __atomic_store(size_t size, void *mem, void *val, int order)
893 // void __atomic_store_N(T *mem, T val, int order)
894 case AtomicExpr::AO__c11_atomic_store:
895 case AtomicExpr::AO__atomic_store:
896 case AtomicExpr::AO__atomic_store_n:
897 LibCallName = "__atomic_store";
898 RetTy = getContext().VoidTy;
900 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
901 MemTy, E->getExprLoc(), sizeChars);
903 // void __atomic_load(size_t size, void *mem, void *return, int order)
904 // T __atomic_load_N(T *mem, int order)
905 case AtomicExpr::AO__c11_atomic_load:
906 case AtomicExpr::AO__atomic_load:
907 case AtomicExpr::AO__atomic_load_n:
908 LibCallName = "__atomic_load";
910 // T __atomic_add_fetch_N(T *mem, T val, int order)
911 // T __atomic_fetch_add_N(T *mem, T val, int order)
912 case AtomicExpr::AO__atomic_add_fetch:
913 PostOp = llvm::Instruction::Add;
915 case AtomicExpr::AO__c11_atomic_fetch_add:
916 case AtomicExpr::AO__atomic_fetch_add:
917 LibCallName = "__atomic_fetch_add";
918 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
919 LoweredMemTy, E->getExprLoc(), sizeChars);
921 // T __atomic_and_fetch_N(T *mem, T val, int order)
922 // T __atomic_fetch_and_N(T *mem, T val, int order)
923 case AtomicExpr::AO__atomic_and_fetch:
924 PostOp = llvm::Instruction::And;
926 case AtomicExpr::AO__c11_atomic_fetch_and:
927 case AtomicExpr::AO__atomic_fetch_and:
928 LibCallName = "__atomic_fetch_and";
929 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
930 MemTy, E->getExprLoc(), sizeChars);
932 // T __atomic_or_fetch_N(T *mem, T val, int order)
933 // T __atomic_fetch_or_N(T *mem, T val, int order)
934 case AtomicExpr::AO__atomic_or_fetch:
935 PostOp = llvm::Instruction::Or;
937 case AtomicExpr::AO__c11_atomic_fetch_or:
938 case AtomicExpr::AO__atomic_fetch_or:
939 LibCallName = "__atomic_fetch_or";
940 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
941 MemTy, E->getExprLoc(), sizeChars);
943 // T __atomic_sub_fetch_N(T *mem, T val, int order)
944 // T __atomic_fetch_sub_N(T *mem, T val, int order)
945 case AtomicExpr::AO__atomic_sub_fetch:
946 PostOp = llvm::Instruction::Sub;
948 case AtomicExpr::AO__c11_atomic_fetch_sub:
949 case AtomicExpr::AO__atomic_fetch_sub:
950 LibCallName = "__atomic_fetch_sub";
951 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
952 LoweredMemTy, E->getExprLoc(), sizeChars);
954 // T __atomic_xor_fetch_N(T *mem, T val, int order)
955 // T __atomic_fetch_xor_N(T *mem, T val, int order)
956 case AtomicExpr::AO__atomic_xor_fetch:
957 PostOp = llvm::Instruction::Xor;
959 case AtomicExpr::AO__c11_atomic_fetch_xor:
960 case AtomicExpr::AO__atomic_fetch_xor:
961 LibCallName = "__atomic_fetch_xor";
962 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
963 MemTy, E->getExprLoc(), sizeChars);
965 // T __atomic_nand_fetch_N(T *mem, T val, int order)
966 // T __atomic_fetch_nand_N(T *mem, T val, int order)
967 case AtomicExpr::AO__atomic_nand_fetch:
968 PostOp = llvm::Instruction::And; // the NOT is special cased below
970 case AtomicExpr::AO__atomic_fetch_nand:
971 LibCallName = "__atomic_fetch_nand";
972 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
973 MemTy, E->getExprLoc(), sizeChars);
977 // Optimized functions have the size in their name.
978 if (UseOptimizedLibcall)
979 LibCallName += "_" + llvm::utostr(Size);
980 // By default, assume we return a value of the atomic type.
982 if (UseOptimizedLibcall) {
983 // Value is returned directly.
984 // The function returns an appropriately sized integer type.
985 RetTy = getContext().getIntTypeForBitwidth(
986 getContext().toBits(sizeChars), /*Signed=*/false);
988 // Value is returned through parameter before the order.
989 RetTy = getContext().VoidTy;
990 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
991 getContext().VoidPtrTy);
994 // order is always the last parameter
995 Args.add(RValue::get(Order),
998 // PostOp is only needed for the atomic_*_fetch operations, and
999 // thus is only needed for and implemented in the
1000 // UseOptimizedLibcall codepath.
1001 assert(UseOptimizedLibcall || !PostOp);
1003 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1004 // The value is returned directly from the libcall.
1008 // The value is returned directly for optimized libcalls but the expr
1009 // provided an out-param.
1010 if (UseOptimizedLibcall && Res.getScalarVal()) {
1011 llvm::Value *ResVal = Res.getScalarVal();
1013 llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1014 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1016 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1017 ResVal = Builder.CreateNot(ResVal);
1019 Builder.CreateStore(
1021 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1024 if (RValTy->isVoidType())
1025 return RValue::get(nullptr);
1027 return convertTempToRValue(
1028 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1029 RValTy, E->getExprLoc());
1032 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1033 E->getOp() == AtomicExpr::AO__atomic_store ||
1034 E->getOp() == AtomicExpr::AO__atomic_store_n;
1035 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1036 E->getOp() == AtomicExpr::AO__atomic_load ||
1037 E->getOp() == AtomicExpr::AO__atomic_load_n;
1039 if (isa<llvm::ConstantInt>(Order)) {
1040 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1042 case AtomicExpr::AO_ABI_memory_order_relaxed:
1043 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1044 Size, llvm::Monotonic);
1046 case AtomicExpr::AO_ABI_memory_order_consume:
1047 case AtomicExpr::AO_ABI_memory_order_acquire:
1049 break; // Avoid crashing on code with undefined behavior
1050 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1051 Size, llvm::Acquire);
1053 case AtomicExpr::AO_ABI_memory_order_release:
1055 break; // Avoid crashing on code with undefined behavior
1056 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1057 Size, llvm::Release);
1059 case AtomicExpr::AO_ABI_memory_order_acq_rel:
1060 if (IsLoad || IsStore)
1061 break; // Avoid crashing on code with undefined behavior
1062 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1063 Size, llvm::AcquireRelease);
1065 case AtomicExpr::AO_ABI_memory_order_seq_cst:
1066 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1067 Size, llvm::SequentiallyConsistent);
1069 default: // invalid order
1070 // We should not ever get here normally, but it's hard to
1071 // enforce that in general.
1074 if (RValTy->isVoidType())
1075 return RValue::get(nullptr);
1077 return convertTempToRValue(
1078 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1079 RValTy, E->getExprLoc());
1082 // Long case, when Order isn't obviously constant.
1084 // Create all the relevant BB's
1085 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1086 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1087 *SeqCstBB = nullptr;
1088 MonotonicBB = createBasicBlock("monotonic", CurFn);
1090 AcquireBB = createBasicBlock("acquire", CurFn);
1092 ReleaseBB = createBasicBlock("release", CurFn);
1093 if (!IsLoad && !IsStore)
1094 AcqRelBB = createBasicBlock("acqrel", CurFn);
1095 SeqCstBB = createBasicBlock("seqcst", CurFn);
1096 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1098 // Create the switch for the split
1099 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1100 // doesn't matter unless someone is crazy enough to use something that
1101 // doesn't fold to a constant for the ordering.
1102 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1103 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1105 // Emit all the different atomics
1106 Builder.SetInsertPoint(MonotonicBB);
1107 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1108 Size, llvm::Monotonic);
1109 Builder.CreateBr(ContBB);
1111 Builder.SetInsertPoint(AcquireBB);
1112 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1113 Size, llvm::Acquire);
1114 Builder.CreateBr(ContBB);
1115 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1117 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1121 Builder.SetInsertPoint(ReleaseBB);
1122 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1123 Size, llvm::Release);
1124 Builder.CreateBr(ContBB);
1125 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1128 if (!IsLoad && !IsStore) {
1129 Builder.SetInsertPoint(AcqRelBB);
1130 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1131 Size, llvm::AcquireRelease);
1132 Builder.CreateBr(ContBB);
1133 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1136 Builder.SetInsertPoint(SeqCstBB);
1137 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1138 Size, llvm::SequentiallyConsistent);
1139 Builder.CreateBr(ContBB);
1140 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1143 // Cleanup and return
1144 Builder.SetInsertPoint(ContBB);
1145 if (RValTy->isVoidType())
1146 return RValue::get(nullptr);
1148 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1149 return convertTempToRValue(
1150 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1151 RValTy, E->getExprLoc());
1154 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1155 unsigned addrspace =
1156 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1157 llvm::IntegerType *ty =
1158 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1159 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1162 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1163 llvm::Type *Ty = Addr.getElementType();
1164 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1165 if (SourceSizeInBits != AtomicSizeInBits) {
1166 Address Tmp = CreateTempAlloca();
1167 CGF.Builder.CreateMemCpy(Tmp, Addr,
1168 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1172 return emitCastToAtomicIntPointer(Addr);
1175 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1176 AggValueSlot resultSlot,
1178 bool asValue) const {
1179 if (LVal.isSimple()) {
1180 if (EvaluationKind == TEK_Aggregate)
1181 return resultSlot.asRValue();
1183 // Drill into the padding structure if we have one.
1185 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1187 // Otherwise, just convert the temporary to an r-value using the
1188 // normal conversion routine.
1189 return CGF.convertTempToRValue(addr, getValueType(), loc);
1192 // Get RValue from temp memory as atomic for non-simple lvalues
1193 return RValue::get(CGF.Builder.CreateLoad(addr));
1194 if (LVal.isBitField())
1195 return CGF.EmitLoadOfBitfieldLValue(
1196 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1197 LVal.getAlignmentSource()));
1198 if (LVal.isVectorElt())
1199 return CGF.EmitLoadOfLValue(
1200 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1201 LVal.getAlignmentSource()), loc);
1202 assert(LVal.isExtVectorElt());
1203 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1204 addr, LVal.getExtVectorElts(), LVal.getType(),
1205 LVal.getAlignmentSource()));
1208 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1209 AggValueSlot ResultSlot,
1211 bool AsValue) const {
1212 // Try not to in some easy cases.
1213 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1214 if (getEvaluationKind() == TEK_Scalar &&
1215 (((!LVal.isBitField() ||
1216 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1219 auto *ValTy = AsValue
1220 ? CGF.ConvertTypeForMem(ValueTy)
1221 : getAtomicAddress().getType()->getPointerElementType();
1222 if (ValTy->isIntegerTy()) {
1223 assert(IntVal->getType() == ValTy && "Different integer types.");
1224 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1225 } else if (ValTy->isPointerTy())
1226 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1227 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1228 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1231 // Create a temporary. This needs to be big enough to hold the
1233 Address Temp = Address::invalid();
1234 bool TempIsVolatile = false;
1235 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1236 assert(!ResultSlot.isIgnored());
1237 Temp = ResultSlot.getAddress();
1238 TempIsVolatile = ResultSlot.isVolatile();
1240 Temp = CreateTempAlloca();
1243 // Slam the integer into the temporary.
1244 Address CastTemp = emitCastToAtomicIntPointer(Temp);
1245 CGF.Builder.CreateStore(IntVal, CastTemp)
1246 ->setVolatile(TempIsVolatile);
1248 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1251 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1252 llvm::AtomicOrdering AO, bool) {
1253 // void __atomic_load(size_t size, void *mem, void *return, int order);
1255 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1256 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1257 CGF.getContext().VoidPtrTy);
1258 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1259 CGF.getContext().VoidPtrTy);
1260 Args.add(RValue::get(
1261 llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1262 CGF.getContext().IntTy);
1263 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1266 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1268 // Okay, we're doing this natively.
1269 Address Addr = getAtomicAddressAsAtomicIntPointer();
1270 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1271 Load->setAtomic(AO);
1273 // Other decoration.
1275 Load->setVolatile(true);
1276 if (LVal.getTBAAInfo())
1277 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1281 /// An LValue is a candidate for having its loads and stores be made atomic if
1282 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1283 /// performing such an operation can be performed without a libcall.
1284 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1285 if (!CGM.getCodeGenOpts().MSVolatile) return false;
1286 AtomicInfo AI(*this, LV);
1287 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1288 // An atomic is inline if we don't need to use a libcall.
1289 bool AtomicIsInline = !AI.shouldUseLibcall();
1290 return IsVolatile && AtomicIsInline;
1293 /// An type is a candidate for having its loads and stores be made atomic if
1294 /// we are operating under /volatile:ms *and* we know the access is volatile and
1295 /// performing such an operation can be performed without a libcall.
1296 bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
1297 bool IsVolatile) const {
1298 // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1299 bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1300 getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1301 return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1304 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1305 AggValueSlot Slot) {
1306 llvm::AtomicOrdering AO;
1307 bool IsVolatile = LV.isVolatileQualified();
1308 if (LV.getType()->isAtomicType()) {
1309 AO = llvm::SequentiallyConsistent;
1314 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1317 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1318 bool AsValue, llvm::AtomicOrdering AO,
1320 // Check whether we should use a library call.
1321 if (shouldUseLibcall()) {
1322 Address TempAddr = Address::invalid();
1323 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1324 assert(getEvaluationKind() == TEK_Aggregate);
1325 TempAddr = ResultSlot.getAddress();
1327 TempAddr = CreateTempAlloca();
1329 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1331 // Okay, turn that back into the original value or whole atomic (for
1332 // non-simple lvalues) type.
1333 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1336 // Okay, we're doing this natively.
1337 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1339 // If we're ignoring an aggregate return, don't do anything.
1340 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1341 return RValue::getAggregate(Address::invalid(), false);
1343 // Okay, turn that back into the original value or atomic (for non-simple
1345 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1348 /// Emit a load from an l-value of atomic type. Note that the r-value
1349 /// we produce is an r-value of the atomic *value* type.
1350 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1351 llvm::AtomicOrdering AO, bool IsVolatile,
1352 AggValueSlot resultSlot) {
1353 AtomicInfo Atomics(*this, src);
1354 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1358 /// Copy an r-value into memory as part of storing to an atomic type.
1359 /// This needs to create a bit-pattern suitable for atomic operations.
1360 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1361 assert(LVal.isSimple());
1362 // If we have an r-value, the rvalue should be of the atomic type,
1363 // which means that the caller is responsible for having zeroed
1364 // any padding. Just do an aggregate copy of that type.
1365 if (rvalue.isAggregate()) {
1366 CGF.EmitAggregateCopy(getAtomicAddress(),
1367 rvalue.getAggregateAddress(),
1369 (rvalue.isVolatileQualified()
1370 || LVal.isVolatileQualified()));
1374 // Okay, otherwise we're copying stuff.
1376 // Zero out the buffer if necessary.
1377 emitMemSetZeroIfNecessary();
1379 // Drill past the padding if present.
1380 LValue TempLVal = projectValue();
1382 // Okay, store the rvalue in.
1383 if (rvalue.isScalar()) {
1384 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1386 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1391 /// Materialize an r-value into memory for the purposes of storing it
1392 /// to an atomic type.
1393 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1394 // Aggregate r-values are already in memory, and EmitAtomicStore
1395 // requires them to be values of the atomic type.
1396 if (rvalue.isAggregate())
1397 return rvalue.getAggregateAddress();
1399 // Otherwise, make a temporary and materialize into it.
1400 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1401 AtomicInfo Atomics(CGF, TempLV);
1402 Atomics.emitCopyIntoMemory(rvalue);
1403 return TempLV.getAddress();
1406 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1407 // If we've got a scalar value of the right size, try to avoid going
1409 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1410 llvm::Value *Value = RVal.getScalarVal();
1411 if (isa<llvm::IntegerType>(Value->getType()))
1412 return CGF.EmitToMemory(Value, ValueTy);
1414 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1415 CGF.getLLVMContext(),
1416 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1417 if (isa<llvm::PointerType>(Value->getType()))
1418 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1419 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1420 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1423 // Otherwise, we need to go through memory.
1424 // Put the r-value in memory.
1425 Address Addr = materializeRValue(RVal);
1427 // Cast the temporary to the atomic int type and pull a value out.
1428 Addr = emitCastToAtomicIntPointer(Addr);
1429 return CGF.Builder.CreateLoad(Addr);
1432 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1433 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1434 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1435 // Do the atomic store.
1436 Address Addr = getAtomicAddressAsAtomicIntPointer();
1437 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1438 ExpectedVal, DesiredVal,
1440 // Other decoration.
1441 Inst->setVolatile(LVal.isVolatileQualified());
1442 Inst->setWeak(IsWeak);
1444 // Okay, turn that back into the original value type.
1445 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1446 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1447 return std::make_pair(PreviousVal, SuccessFailureVal);
1451 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1452 llvm::Value *DesiredAddr,
1453 llvm::AtomicOrdering Success,
1454 llvm::AtomicOrdering Failure) {
1455 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1456 // void *desired, int success, int failure);
1458 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1459 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1460 CGF.getContext().VoidPtrTy);
1461 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1462 CGF.getContext().VoidPtrTy);
1463 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1464 CGF.getContext().VoidPtrTy);
1465 Args.add(RValue::get(llvm::ConstantInt::get(
1466 CGF.IntTy, translateAtomicOrdering(Success))),
1467 CGF.getContext().IntTy);
1468 Args.add(RValue::get(llvm::ConstantInt::get(
1469 CGF.IntTy, translateAtomicOrdering(Failure))),
1470 CGF.getContext().IntTy);
1471 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1472 CGF.getContext().BoolTy, Args);
1474 return SuccessFailureRVal.getScalarVal();
1477 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1478 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1479 llvm::AtomicOrdering Failure, bool IsWeak) {
1480 if (Failure >= Success)
1481 // Don't assert on undefined behavior.
1482 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1484 // Check whether we should use a library call.
1485 if (shouldUseLibcall()) {
1486 // Produce a source address.
1487 Address ExpectedAddr = materializeRValue(Expected);
1488 Address DesiredAddr = materializeRValue(Desired);
1489 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1490 DesiredAddr.getPointer(),
1492 return std::make_pair(
1493 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1494 SourceLocation(), /*AsValue=*/false),
1498 // If we've got a scalar value of the right size, try to avoid going
1500 auto *ExpectedVal = convertRValueToInt(Expected);
1501 auto *DesiredVal = convertRValueToInt(Desired);
1502 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1504 return std::make_pair(
1505 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1506 SourceLocation(), /*AsValue=*/false),
1511 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1512 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1513 Address DesiredAddr) {
1515 LValue AtomicLVal = Atomics.getAtomicLValue();
1517 if (AtomicLVal.isSimple()) {
1519 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1521 // Build new lvalue for temp address
1522 Address Ptr = Atomics.materializeRValue(OldRVal);
1524 if (AtomicLVal.isBitField()) {
1526 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1527 AtomicLVal.getType(),
1528 AtomicLVal.getAlignmentSource());
1530 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1531 AtomicLVal.getType(),
1532 AtomicLVal.getAlignmentSource());
1533 } else if (AtomicLVal.isVectorElt()) {
1534 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1535 AtomicLVal.getType(),
1536 AtomicLVal.getAlignmentSource());
1537 DesiredLVal = LValue::MakeVectorElt(
1538 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1539 AtomicLVal.getAlignmentSource());
1541 assert(AtomicLVal.isExtVectorElt());
1542 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1543 AtomicLVal.getType(),
1544 AtomicLVal.getAlignmentSource());
1545 DesiredLVal = LValue::MakeExtVectorElt(
1546 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1547 AtomicLVal.getAlignmentSource());
1549 UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1550 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1551 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1553 // Store new value in the corresponding memory area
1554 RValue NewRVal = UpdateOp(UpRVal);
1555 if (NewRVal.isScalar()) {
1556 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1558 assert(NewRVal.isComplex());
1559 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1564 void AtomicInfo::EmitAtomicUpdateLibcall(
1565 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1567 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1569 Address ExpectedAddr = CreateTempAlloca();
1571 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1572 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1573 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1574 CGF.EmitBlock(ContBB);
1575 Address DesiredAddr = CreateTempAlloca();
1576 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1577 requiresMemSetZero(getAtomicAddress().getElementType())) {
1578 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1579 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1581 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1582 AggValueSlot::ignored(),
1583 SourceLocation(), /*AsValue=*/false);
1584 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1586 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1587 DesiredAddr.getPointer(),
1589 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1590 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1593 void AtomicInfo::EmitAtomicUpdateOp(
1594 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1596 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1598 // Do the atomic load.
1599 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1600 // For non-simple lvalues perform compare-and-swap procedure.
1601 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1602 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1603 auto *CurBB = CGF.Builder.GetInsertBlock();
1604 CGF.EmitBlock(ContBB);
1605 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1606 /*NumReservedValues=*/2);
1607 PHI->addIncoming(OldVal, CurBB);
1608 Address NewAtomicAddr = CreateTempAlloca();
1609 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1610 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1611 requiresMemSetZero(getAtomicAddress().getElementType())) {
1612 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1614 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1615 SourceLocation(), /*AsValue=*/false);
1616 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1617 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1618 // Try to write new value using cmpxchg operation
1619 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1620 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1621 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1622 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1625 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1626 RValue UpdateRVal, Address DesiredAddr) {
1627 LValue AtomicLVal = Atomics.getAtomicLValue();
1629 // Build new lvalue for temp address
1630 if (AtomicLVal.isBitField()) {
1632 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1633 AtomicLVal.getType(),
1634 AtomicLVal.getAlignmentSource());
1635 } else if (AtomicLVal.isVectorElt()) {
1637 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1638 AtomicLVal.getType(),
1639 AtomicLVal.getAlignmentSource());
1641 assert(AtomicLVal.isExtVectorElt());
1642 DesiredLVal = LValue::MakeExtVectorElt(
1643 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1644 AtomicLVal.getAlignmentSource());
1646 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1647 // Store new value in the corresponding memory area
1648 assert(UpdateRVal.isScalar());
1649 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1652 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1653 RValue UpdateRVal, bool IsVolatile) {
1654 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1656 Address ExpectedAddr = CreateTempAlloca();
1658 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1659 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1660 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1661 CGF.EmitBlock(ContBB);
1662 Address DesiredAddr = CreateTempAlloca();
1663 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1664 requiresMemSetZero(getAtomicAddress().getElementType())) {
1665 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1666 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1668 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1670 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1671 DesiredAddr.getPointer(),
1673 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1674 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1677 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1679 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1681 // Do the atomic load.
1682 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1683 // For non-simple lvalues perform compare-and-swap procedure.
1684 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1685 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1686 auto *CurBB = CGF.Builder.GetInsertBlock();
1687 CGF.EmitBlock(ContBB);
1688 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1689 /*NumReservedValues=*/2);
1690 PHI->addIncoming(OldVal, CurBB);
1691 Address NewAtomicAddr = CreateTempAlloca();
1692 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1693 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1694 requiresMemSetZero(getAtomicAddress().getElementType())) {
1695 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1697 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1698 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1699 // Try to write new value using cmpxchg operation
1700 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1701 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1702 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1703 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1706 void AtomicInfo::EmitAtomicUpdate(
1707 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1709 if (shouldUseLibcall()) {
1710 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1712 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1716 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1718 if (shouldUseLibcall()) {
1719 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1721 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1725 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1727 bool IsVolatile = lvalue.isVolatileQualified();
1728 llvm::AtomicOrdering AO;
1729 if (lvalue.getType()->isAtomicType()) {
1730 AO = llvm::SequentiallyConsistent;
1735 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1738 /// Emit a store to an l-value of atomic type.
1740 /// Note that the r-value is expected to be an r-value *of the atomic
1741 /// type*; this means that for aggregate r-values, it should include
1742 /// storage for any padding that was necessary.
1743 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1744 llvm::AtomicOrdering AO, bool IsVolatile,
1746 // If this is an aggregate r-value, it should agree in type except
1747 // maybe for address-space qualification.
1748 assert(!rvalue.isAggregate() ||
1749 rvalue.getAggregateAddress().getElementType()
1750 == dest.getAddress().getElementType());
1752 AtomicInfo atomics(*this, dest);
1753 LValue LVal = atomics.getAtomicLValue();
1755 // If this is an initialization, just put the value there normally.
1756 if (LVal.isSimple()) {
1758 atomics.emitCopyIntoMemory(rvalue);
1762 // Check whether we should use a library call.
1763 if (atomics.shouldUseLibcall()) {
1764 // Produce a source address.
1765 Address srcAddr = atomics.materializeRValue(rvalue);
1767 // void __atomic_store(size_t size, void *mem, void *val, int order)
1769 args.add(RValue::get(atomics.getAtomicSizeValue()),
1770 getContext().getSizeType());
1771 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1772 getContext().VoidPtrTy);
1773 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1774 getContext().VoidPtrTy);
1775 args.add(RValue::get(llvm::ConstantInt::get(
1776 IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1777 getContext().IntTy);
1778 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1782 // Okay, we're doing this natively.
1783 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1785 // Do the atomic store.
1787 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1788 intValue = Builder.CreateIntCast(
1789 intValue, addr.getElementType(), /*isSigned=*/false);
1790 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1792 // Initializations don't need to be atomic.
1794 store->setAtomic(AO);
1796 // Other decoration.
1798 store->setVolatile(true);
1799 if (dest.getTBAAInfo())
1800 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1804 // Emit simple atomic update operation.
1805 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1808 /// Emit a compare-and-exchange op for atomic type.
1810 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1811 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1812 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1813 AggValueSlot Slot) {
1814 // If this is an aggregate r-value, it should agree in type except
1815 // maybe for address-space qualification.
1816 assert(!Expected.isAggregate() ||
1817 Expected.getAggregateAddress().getElementType() ==
1818 Obj.getAddress().getElementType());
1819 assert(!Desired.isAggregate() ||
1820 Desired.getAggregateAddress().getElementType() ==
1821 Obj.getAddress().getElementType());
1822 AtomicInfo Atomics(*this, Obj);
1824 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1828 void CodeGenFunction::EmitAtomicUpdate(
1829 LValue LVal, llvm::AtomicOrdering AO,
1830 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1831 AtomicInfo Atomics(*this, LVal);
1832 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1835 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1836 AtomicInfo atomics(*this, dest);
1838 switch (atomics.getEvaluationKind()) {
1840 llvm::Value *value = EmitScalarExpr(init);
1841 atomics.emitCopyIntoMemory(RValue::get(value));
1846 ComplexPairTy value = EmitComplexExpr(init);
1847 atomics.emitCopyIntoMemory(RValue::getComplex(value));
1851 case TEK_Aggregate: {
1852 // Fix up the destination if the initializer isn't an expression
1854 bool Zeroed = false;
1855 if (!init->getType()->isAtomicType()) {
1856 Zeroed = atomics.emitMemSetZeroIfNecessary();
1857 dest = atomics.projectValue();
1860 // Evaluate the expression directly into the destination.
1861 AggValueSlot slot = AggValueSlot::forLValue(dest,
1862 AggValueSlot::IsNotDestructed,
1863 AggValueSlot::DoesNotNeedGCBarriers,
1864 AggValueSlot::IsNotAliased,
1865 Zeroed ? AggValueSlot::IsZeroed :
1866 AggValueSlot::IsNotZeroed);
1868 EmitAggExpr(init, slot);
1872 llvm_unreachable("bad evaluation kind");