1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the code for emitting atomic operations.
12 //===----------------------------------------------------------------------===//
14 #include "CodeGenFunction.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Operator.h"
25 using namespace clang;
26 using namespace CodeGen;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
37 CharUnits LValueAlign;
38 TypeEvaluationKind EvaluationKind;
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .RoundUpToAlignment(lvalue.getAlignment()));
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88 VoidPtrAddr, OffsetInChars.getQuantity());
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92 "atomic_bitfield_base");
95 BFI.StorageSize = AtomicSizeInBits;
96 LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
97 lvalue.getAlignment());
98 LVal.setTBAAInfo(lvalue.getTBAAInfo());
99 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100 if (AtomicTy.isNull()) {
103 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
105 /*IndexTypeQuals=*/0);
107 AtomicAlign = ValueAlign = lvalue.getAlignment();
108 } else if (lvalue.isVectorElt()) {
109 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
110 ValueSizeInBits = C.getTypeSize(ValueTy);
111 AtomicTy = lvalue.getType();
112 AtomicSizeInBits = C.getTypeSize(AtomicTy);
113 AtomicAlign = ValueAlign = lvalue.getAlignment();
116 assert(lvalue.isExtVectorElt());
117 ValueTy = lvalue.getType();
118 ValueSizeInBits = C.getTypeSize(ValueTy);
119 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120 lvalue.getType(), lvalue.getExtVectorAddr()
122 ->getPointerElementType()
123 ->getVectorNumElements());
124 AtomicSizeInBits = C.getTypeSize(AtomicTy);
125 AtomicAlign = ValueAlign = lvalue.getAlignment();
128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
132 QualType getAtomicType() const { return AtomicTy; }
133 QualType getValueType() const { return ValueTy; }
134 CharUnits getAtomicAlignment() const { return AtomicAlign; }
135 CharUnits getValueAlignment() const { return ValueAlign; }
136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139 bool shouldUseLibcall() const { return UseLibcall; }
140 const LValue &getAtomicLValue() const { return LVal; }
141 llvm::Value *getAtomicAddress() const {
143 return LVal.getAddress();
144 else if (LVal.isBitField())
145 return LVal.getBitFieldAddr();
146 else if (LVal.isVectorElt())
147 return LVal.getVectorAddr();
148 assert(LVal.isExtVectorElt());
149 return LVal.getExtVectorAddr();
152 /// Is the atomic size larger than the underlying value type?
154 /// Note that the absence of padding does not mean that atomic
155 /// objects are completely interchangeable with non-atomic
156 /// objects: we might have promoted the alignment of a type
157 /// without making it bigger.
158 bool hasPadding() const {
159 return (ValueSizeInBits != AtomicSizeInBits);
162 bool emitMemSetZeroIfNecessary() const;
164 llvm::Value *getAtomicSizeValue() const {
165 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
166 return CGF.CGM.getSize(size);
169 /// Cast the given pointer to an integer pointer suitable for
170 /// atomic operations.
171 llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
173 /// Turn an atomic-layout object into an r-value.
174 RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
175 SourceLocation loc, bool AsValue) const;
177 /// \brief Converts a rvalue to integer value.
178 llvm::Value *convertRValueToInt(RValue RVal) const;
180 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
181 AggValueSlot ResultSlot,
182 SourceLocation Loc, bool AsValue) const;
184 /// Copy an atomic r-value into atomic-layout memory.
185 void emitCopyIntoMemory(RValue rvalue) const;
187 /// Project an l-value down to the value field.
188 LValue projectValue() const {
189 assert(LVal.isSimple());
190 llvm::Value *addr = getAtomicAddress();
192 addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
194 return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
195 CGF.getContext(), LVal.getTBAAInfo());
198 /// \brief Emits atomic load.
199 /// \returns Loaded value.
200 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
201 bool AsValue, llvm::AtomicOrdering AO,
204 /// \brief Emits atomic compare-and-exchange sequence.
205 /// \param Expected Expected value.
206 /// \param Desired Desired value.
207 /// \param Success Atomic ordering for success operation.
208 /// \param Failure Atomic ordering for failed operation.
209 /// \param IsWeak true if atomic operation is weak, false otherwise.
210 /// \returns Pair of values: previous value from storage (value type) and
211 /// boolean flag (i1 type) with true if success and false otherwise.
212 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
213 RValue Expected, RValue Desired,
214 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
215 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
216 bool IsWeak = false);
218 /// \brief Emits atomic update.
219 /// \param AO Atomic ordering.
220 /// \param UpdateOp Update operation for the current lvalue.
221 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
222 const llvm::function_ref<RValue(RValue)> &UpdateOp,
224 /// \brief Emits atomic update.
225 /// \param AO Atomic ordering.
226 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
229 /// Materialize an atomic r-value in atomic-layout memory.
230 llvm::Value *materializeRValue(RValue rvalue) const;
232 /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
234 static AtomicExpr::AtomicOrderingKind
235 translateAtomicOrdering(const llvm::AtomicOrdering AO);
238 bool requiresMemSetZero(llvm::Type *type) const;
240 /// \brief Creates temp alloca for intermediate operations on atomic value.
241 llvm::Value *CreateTempAlloca() const;
243 /// \brief Emits atomic load as a libcall.
244 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
245 llvm::AtomicOrdering AO, bool IsVolatile);
246 /// \brief Emits atomic load as LLVM instruction.
247 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
248 /// \brief Emits atomic compare-and-exchange op as a libcall.
249 llvm::Value *EmitAtomicCompareExchangeLibcall(
250 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
251 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
252 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
253 /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
254 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
255 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
256 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
257 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
258 bool IsWeak = false);
259 /// \brief Emit atomic update as libcalls.
261 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
262 const llvm::function_ref<RValue(RValue)> &UpdateOp,
264 /// \brief Emit atomic update as LLVM instructions.
265 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
266 const llvm::function_ref<RValue(RValue)> &UpdateOp,
268 /// \brief Emit atomic update as libcalls.
269 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
271 /// \brief Emit atomic update as LLVM instructions.
272 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
277 AtomicExpr::AtomicOrderingKind
278 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
280 case llvm::Unordered:
281 case llvm::NotAtomic:
282 case llvm::Monotonic:
283 return AtomicExpr::AO_ABI_memory_order_relaxed;
285 return AtomicExpr::AO_ABI_memory_order_acquire;
287 return AtomicExpr::AO_ABI_memory_order_release;
288 case llvm::AcquireRelease:
289 return AtomicExpr::AO_ABI_memory_order_acq_rel;
290 case llvm::SequentiallyConsistent:
291 return AtomicExpr::AO_ABI_memory_order_seq_cst;
293 llvm_unreachable("Unhandled AtomicOrdering");
296 llvm::Value *AtomicInfo::CreateTempAlloca() const {
297 auto *TempAlloca = CGF.CreateMemTemp(
298 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301 TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
302 // Cast to pointer to value type for bitfields.
303 if (LVal.isBitField())
304 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
305 TempAlloca, getAtomicAddress()->getType());
309 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
313 const CGFunctionInfo &fnInfo =
314 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
315 FunctionType::ExtInfo(), RequiredArgs::All);
316 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
317 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
318 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
321 /// Does a store of the given IR type modify the full expected width?
322 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
323 uint64_t expectedSize) {
324 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
327 /// Does the atomic type require memsetting to zero before initialization?
329 /// The IR type is provided as a way of making certain queries faster.
330 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
331 // If the atomic type has size padding, we definitely need a memset.
332 if (hasPadding()) return true;
334 // Otherwise, do some simple heuristics to try to avoid it:
335 switch (getEvaluationKind()) {
336 // For scalars and complexes, check whether the store size of the
337 // type uses the full size.
339 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
341 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
342 AtomicSizeInBits / 2);
344 // Padding in structs has an undefined bit pattern. User beware.
348 llvm_unreachable("bad evaluation kind");
351 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
352 assert(LVal.isSimple());
353 llvm::Value *addr = LVal.getAddress();
354 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
357 CGF.Builder.CreateMemSet(
358 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
359 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
360 LVal.getAlignment().getQuantity());
364 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
365 llvm::Value *Dest, llvm::Value *Ptr,
366 llvm::Value *Val1, llvm::Value *Val2,
367 uint64_t Size, unsigned Align,
368 llvm::AtomicOrdering SuccessOrder,
369 llvm::AtomicOrdering FailureOrder) {
370 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
371 llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
372 Expected->setAlignment(Align);
373 llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
374 Desired->setAlignment(Align);
376 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
377 Ptr, Expected, Desired, SuccessOrder, FailureOrder);
378 Pair->setVolatile(E->isVolatile());
379 Pair->setWeak(IsWeak);
381 // Cmp holds the result of the compare-exchange operation: true on success,
383 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
384 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
386 // This basic block is used to hold the store instruction if the operation
388 llvm::BasicBlock *StoreExpectedBB =
389 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
391 // This basic block is the exit point of the operation, we should end up
392 // here regardless of whether or not the operation succeeded.
393 llvm::BasicBlock *ContinueBB =
394 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
396 // Update Expected if Expected isn't equal to Old, otherwise branch to the
398 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
400 CGF.Builder.SetInsertPoint(StoreExpectedBB);
401 // Update the memory at Expected with Old's value.
402 llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
403 StoreExpected->setAlignment(Align);
404 // Finally, branch to the exit point.
405 CGF.Builder.CreateBr(ContinueBB);
407 CGF.Builder.SetInsertPoint(ContinueBB);
408 // Update the memory at Dest with Cmp's value.
409 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
413 /// Given an ordering required on success, emit all possible cmpxchg
414 /// instructions to cope with the provided (but possibly only dynamically known)
416 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
417 bool IsWeak, llvm::Value *Dest,
418 llvm::Value *Ptr, llvm::Value *Val1,
420 llvm::Value *FailureOrderVal,
421 uint64_t Size, unsigned Align,
422 llvm::AtomicOrdering SuccessOrder) {
423 llvm::AtomicOrdering FailureOrder;
424 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
425 switch (FO->getSExtValue()) {
427 FailureOrder = llvm::Monotonic;
429 case AtomicExpr::AO_ABI_memory_order_consume:
430 case AtomicExpr::AO_ABI_memory_order_acquire:
431 FailureOrder = llvm::Acquire;
433 case AtomicExpr::AO_ABI_memory_order_seq_cst:
434 FailureOrder = llvm::SequentiallyConsistent;
437 if (FailureOrder >= SuccessOrder) {
438 // Don't assert on undefined behaviour.
440 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
442 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
443 SuccessOrder, FailureOrder);
447 // Create all the relevant BB's
448 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
450 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
451 if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
452 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
453 if (SuccessOrder == llvm::SequentiallyConsistent)
454 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
456 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
458 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
460 // Emit all the different atomics
462 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
463 // doesn't matter unless someone is crazy enough to use something that
464 // doesn't fold to a constant for the ordering.
465 CGF.Builder.SetInsertPoint(MonotonicBB);
466 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
467 Size, Align, SuccessOrder, llvm::Monotonic);
468 CGF.Builder.CreateBr(ContBB);
471 CGF.Builder.SetInsertPoint(AcquireBB);
472 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
473 Size, Align, SuccessOrder, llvm::Acquire);
474 CGF.Builder.CreateBr(ContBB);
475 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
477 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
481 CGF.Builder.SetInsertPoint(SeqCstBB);
482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
483 Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
484 CGF.Builder.CreateBr(ContBB);
485 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
489 CGF.Builder.SetInsertPoint(ContBB);
492 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
493 llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
494 llvm::Value *IsWeak, llvm::Value *FailureOrder,
495 uint64_t Size, unsigned Align,
496 llvm::AtomicOrdering Order) {
497 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
498 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
500 switch (E->getOp()) {
501 case AtomicExpr::AO__c11_atomic_init:
502 llvm_unreachable("Already handled!");
504 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
505 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
506 FailureOrder, Size, Align, Order);
508 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
509 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
510 FailureOrder, Size, Align, Order);
512 case AtomicExpr::AO__atomic_compare_exchange:
513 case AtomicExpr::AO__atomic_compare_exchange_n: {
514 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
515 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
516 Val1, Val2, FailureOrder, Size, Align, Order);
518 // Create all the relevant BB's
519 llvm::BasicBlock *StrongBB =
520 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
521 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
522 llvm::BasicBlock *ContBB =
523 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
525 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
526 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
528 CGF.Builder.SetInsertPoint(StrongBB);
529 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
530 FailureOrder, Size, Align, Order);
531 CGF.Builder.CreateBr(ContBB);
533 CGF.Builder.SetInsertPoint(WeakBB);
534 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
535 FailureOrder, Size, Align, Order);
536 CGF.Builder.CreateBr(ContBB);
538 CGF.Builder.SetInsertPoint(ContBB);
542 case AtomicExpr::AO__c11_atomic_load:
543 case AtomicExpr::AO__atomic_load_n:
544 case AtomicExpr::AO__atomic_load: {
545 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
546 Load->setAtomic(Order);
547 Load->setAlignment(Size);
548 Load->setVolatile(E->isVolatile());
549 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
550 StoreDest->setAlignment(Align);
554 case AtomicExpr::AO__c11_atomic_store:
555 case AtomicExpr::AO__atomic_store:
556 case AtomicExpr::AO__atomic_store_n: {
557 assert(!Dest && "Store does not return a value");
558 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
559 LoadVal1->setAlignment(Align);
560 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
561 Store->setAtomic(Order);
562 Store->setAlignment(Size);
563 Store->setVolatile(E->isVolatile());
567 case AtomicExpr::AO__c11_atomic_exchange:
568 case AtomicExpr::AO__atomic_exchange_n:
569 case AtomicExpr::AO__atomic_exchange:
570 Op = llvm::AtomicRMWInst::Xchg;
573 case AtomicExpr::AO__atomic_add_fetch:
574 PostOp = llvm::Instruction::Add;
576 case AtomicExpr::AO__c11_atomic_fetch_add:
577 case AtomicExpr::AO__atomic_fetch_add:
578 Op = llvm::AtomicRMWInst::Add;
581 case AtomicExpr::AO__atomic_sub_fetch:
582 PostOp = llvm::Instruction::Sub;
584 case AtomicExpr::AO__c11_atomic_fetch_sub:
585 case AtomicExpr::AO__atomic_fetch_sub:
586 Op = llvm::AtomicRMWInst::Sub;
589 case AtomicExpr::AO__atomic_and_fetch:
590 PostOp = llvm::Instruction::And;
592 case AtomicExpr::AO__c11_atomic_fetch_and:
593 case AtomicExpr::AO__atomic_fetch_and:
594 Op = llvm::AtomicRMWInst::And;
597 case AtomicExpr::AO__atomic_or_fetch:
598 PostOp = llvm::Instruction::Or;
600 case AtomicExpr::AO__c11_atomic_fetch_or:
601 case AtomicExpr::AO__atomic_fetch_or:
602 Op = llvm::AtomicRMWInst::Or;
605 case AtomicExpr::AO__atomic_xor_fetch:
606 PostOp = llvm::Instruction::Xor;
608 case AtomicExpr::AO__c11_atomic_fetch_xor:
609 case AtomicExpr::AO__atomic_fetch_xor:
610 Op = llvm::AtomicRMWInst::Xor;
613 case AtomicExpr::AO__atomic_nand_fetch:
614 PostOp = llvm::Instruction::And;
616 case AtomicExpr::AO__atomic_fetch_nand:
617 Op = llvm::AtomicRMWInst::Nand;
621 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
622 LoadVal1->setAlignment(Align);
623 llvm::AtomicRMWInst *RMWI =
624 CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
625 RMWI->setVolatile(E->isVolatile());
627 // For __atomic_*_fetch operations, perform the operation again to
628 // determine the value which was written.
629 llvm::Value *Result = RMWI;
631 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
632 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
633 Result = CGF.Builder.CreateNot(Result);
634 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
635 StoreDest->setAlignment(Align);
638 // This function emits any expression (scalar, complex, or aggregate)
639 // into a temporary alloca.
641 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
642 llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
643 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
649 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
650 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
651 SourceLocation Loc, CharUnits SizeInChars) {
652 if (UseOptimizedLibcall) {
653 // Load value and pass it to the function directly.
654 unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
655 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
657 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
658 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
659 SizeInBits)->getPointerTo();
660 Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
661 Align, CGF.getContext().getPointerType(ValTy),
663 // Coerce the value into an appropriately sized integer type.
664 Args.add(RValue::get(Val), ValTy);
666 // Non-optimized functions always take a reference.
667 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
668 CGF.getContext().VoidPtrTy);
672 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
673 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
674 QualType MemTy = AtomicTy;
675 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
676 MemTy = AT->getValueType();
677 CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
678 uint64_t Size = sizeChars.getQuantity();
679 CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
680 unsigned Align = alignChars.getQuantity();
681 unsigned MaxInlineWidthInBits =
682 getTarget().getMaxAtomicInlineWidth();
683 bool UseLibcall = (Size != Align ||
684 getContext().toBits(sizeChars) > MaxInlineWidthInBits);
686 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
688 llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
690 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
691 assert(!Dest && "Init does not return a value");
692 LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
693 EmitAtomicInit(E->getVal1(), lvalue);
694 return RValue::get(nullptr);
697 llvm::Value *Order = EmitScalarExpr(E->getOrder());
699 switch (E->getOp()) {
700 case AtomicExpr::AO__c11_atomic_init:
701 llvm_unreachable("Already handled!");
703 case AtomicExpr::AO__c11_atomic_load:
704 case AtomicExpr::AO__atomic_load_n:
707 case AtomicExpr::AO__atomic_load:
708 Dest = EmitScalarExpr(E->getVal1());
711 case AtomicExpr::AO__atomic_store:
712 Val1 = EmitScalarExpr(E->getVal1());
715 case AtomicExpr::AO__atomic_exchange:
716 Val1 = EmitScalarExpr(E->getVal1());
717 Dest = EmitScalarExpr(E->getVal2());
720 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
721 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
722 case AtomicExpr::AO__atomic_compare_exchange_n:
723 case AtomicExpr::AO__atomic_compare_exchange:
724 Val1 = EmitScalarExpr(E->getVal1());
725 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
726 Val2 = EmitScalarExpr(E->getVal2());
728 Val2 = EmitValToTemp(*this, E->getVal2());
729 OrderFail = EmitScalarExpr(E->getOrderFail());
730 if (E->getNumSubExprs() == 6)
731 IsWeak = EmitScalarExpr(E->getWeak());
734 case AtomicExpr::AO__c11_atomic_fetch_add:
735 case AtomicExpr::AO__c11_atomic_fetch_sub:
736 if (MemTy->isPointerType()) {
737 // For pointer arithmetic, we're required to do a bit of math:
738 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
739 // ... but only for the C11 builtins. The GNU builtins expect the
740 // user to multiply by sizeof(T).
741 QualType Val1Ty = E->getVal1()->getType();
742 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
743 CharUnits PointeeIncAmt =
744 getContext().getTypeSizeInChars(MemTy->getPointeeType());
745 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
746 Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
747 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
751 case AtomicExpr::AO__atomic_fetch_add:
752 case AtomicExpr::AO__atomic_fetch_sub:
753 case AtomicExpr::AO__atomic_add_fetch:
754 case AtomicExpr::AO__atomic_sub_fetch:
755 case AtomicExpr::AO__c11_atomic_store:
756 case AtomicExpr::AO__c11_atomic_exchange:
757 case AtomicExpr::AO__atomic_store_n:
758 case AtomicExpr::AO__atomic_exchange_n:
759 case AtomicExpr::AO__c11_atomic_fetch_and:
760 case AtomicExpr::AO__c11_atomic_fetch_or:
761 case AtomicExpr::AO__c11_atomic_fetch_xor:
762 case AtomicExpr::AO__atomic_fetch_and:
763 case AtomicExpr::AO__atomic_fetch_or:
764 case AtomicExpr::AO__atomic_fetch_xor:
765 case AtomicExpr::AO__atomic_fetch_nand:
766 case AtomicExpr::AO__atomic_and_fetch:
767 case AtomicExpr::AO__atomic_or_fetch:
768 case AtomicExpr::AO__atomic_xor_fetch:
769 case AtomicExpr::AO__atomic_nand_fetch:
770 Val1 = EmitValToTemp(*this, E->getVal1());
774 QualType RValTy = E->getType().getUnqualifiedType();
777 if (!RValTy->isVoidType() && !Dest) {
778 Dest = CreateMemTemp(RValTy, ".atomicdst");
783 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
785 bool UseOptimizedLibcall = false;
786 switch (E->getOp()) {
787 case AtomicExpr::AO__c11_atomic_fetch_add:
788 case AtomicExpr::AO__atomic_fetch_add:
789 case AtomicExpr::AO__c11_atomic_fetch_and:
790 case AtomicExpr::AO__atomic_fetch_and:
791 case AtomicExpr::AO__c11_atomic_fetch_or:
792 case AtomicExpr::AO__atomic_fetch_or:
793 case AtomicExpr::AO__c11_atomic_fetch_sub:
794 case AtomicExpr::AO__atomic_fetch_sub:
795 case AtomicExpr::AO__c11_atomic_fetch_xor:
796 case AtomicExpr::AO__atomic_fetch_xor:
797 // For these, only library calls for certain sizes exist.
798 UseOptimizedLibcall = true;
801 // Only use optimized library calls for sizes for which they exist.
802 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
803 UseOptimizedLibcall = true;
808 if (!UseOptimizedLibcall) {
809 // For non-optimized library calls, the size is the first parameter
810 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
811 getContext().getSizeType());
813 // Atomic address is the first or second parameter
814 Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
816 std::string LibCallName;
817 QualType LoweredMemTy =
818 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
820 bool HaveRetTy = false;
821 switch (E->getOp()) {
822 // There is only one libcall for compare an exchange, because there is no
823 // optimisation benefit possible from a libcall version of a weak compare
825 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
826 // void *desired, int success, int failure)
827 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
828 // int success, int failure)
829 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
830 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
831 case AtomicExpr::AO__atomic_compare_exchange:
832 case AtomicExpr::AO__atomic_compare_exchange_n:
833 LibCallName = "__atomic_compare_exchange";
834 RetTy = getContext().BoolTy;
836 Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
837 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
838 E->getExprLoc(), sizeChars);
839 Args.add(RValue::get(Order), getContext().IntTy);
842 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
844 // T __atomic_exchange_N(T *mem, T val, int order)
845 case AtomicExpr::AO__c11_atomic_exchange:
846 case AtomicExpr::AO__atomic_exchange_n:
847 case AtomicExpr::AO__atomic_exchange:
848 LibCallName = "__atomic_exchange";
849 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
850 E->getExprLoc(), sizeChars);
852 // void __atomic_store(size_t size, void *mem, void *val, int order)
853 // void __atomic_store_N(T *mem, T val, int order)
854 case AtomicExpr::AO__c11_atomic_store:
855 case AtomicExpr::AO__atomic_store:
856 case AtomicExpr::AO__atomic_store_n:
857 LibCallName = "__atomic_store";
858 RetTy = getContext().VoidTy;
860 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
861 E->getExprLoc(), sizeChars);
863 // void __atomic_load(size_t size, void *mem, void *return, int order)
864 // T __atomic_load_N(T *mem, int order)
865 case AtomicExpr::AO__c11_atomic_load:
866 case AtomicExpr::AO__atomic_load:
867 case AtomicExpr::AO__atomic_load_n:
868 LibCallName = "__atomic_load";
870 // T __atomic_fetch_add_N(T *mem, T val, int order)
871 case AtomicExpr::AO__c11_atomic_fetch_add:
872 case AtomicExpr::AO__atomic_fetch_add:
873 LibCallName = "__atomic_fetch_add";
874 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
875 E->getExprLoc(), sizeChars);
877 // T __atomic_fetch_and_N(T *mem, T val, int order)
878 case AtomicExpr::AO__c11_atomic_fetch_and:
879 case AtomicExpr::AO__atomic_fetch_and:
880 LibCallName = "__atomic_fetch_and";
881 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
882 E->getExprLoc(), sizeChars);
884 // T __atomic_fetch_or_N(T *mem, T val, int order)
885 case AtomicExpr::AO__c11_atomic_fetch_or:
886 case AtomicExpr::AO__atomic_fetch_or:
887 LibCallName = "__atomic_fetch_or";
888 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
889 E->getExprLoc(), sizeChars);
891 // T __atomic_fetch_sub_N(T *mem, T val, int order)
892 case AtomicExpr::AO__c11_atomic_fetch_sub:
893 case AtomicExpr::AO__atomic_fetch_sub:
894 LibCallName = "__atomic_fetch_sub";
895 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
896 E->getExprLoc(), sizeChars);
898 // T __atomic_fetch_xor_N(T *mem, T val, int order)
899 case AtomicExpr::AO__c11_atomic_fetch_xor:
900 case AtomicExpr::AO__atomic_fetch_xor:
901 LibCallName = "__atomic_fetch_xor";
902 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
903 E->getExprLoc(), sizeChars);
905 default: return EmitUnsupportedRValue(E, "atomic library call");
908 // Optimized functions have the size in their name.
909 if (UseOptimizedLibcall)
910 LibCallName += "_" + llvm::utostr(Size);
911 // By default, assume we return a value of the atomic type.
913 if (UseOptimizedLibcall) {
914 // Value is returned directly.
915 // The function returns an appropriately sized integer type.
916 RetTy = getContext().getIntTypeForBitwidth(
917 getContext().toBits(sizeChars), /*Signed=*/false);
919 // Value is returned through parameter before the order.
920 RetTy = getContext().VoidTy;
921 Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
924 // order is always the last parameter
925 Args.add(RValue::get(Order),
928 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
929 // The value is returned directly from the libcall.
930 if (HaveRetTy && !RetTy->isVoidType())
932 // The value is returned via an explicit out param.
933 if (RetTy->isVoidType())
934 return RValue::get(nullptr);
935 // The value is returned directly for optimized libcalls but the caller is
936 // expected an out-param.
937 if (UseOptimizedLibcall) {
938 llvm::Value *ResVal = Res.getScalarVal();
939 llvm::StoreInst *StoreDest = Builder.CreateStore(
941 Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
942 StoreDest->setAlignment(Align);
944 return convertTempToRValue(Dest, RValTy, E->getExprLoc());
947 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
948 E->getOp() == AtomicExpr::AO__atomic_store ||
949 E->getOp() == AtomicExpr::AO__atomic_store_n;
950 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
951 E->getOp() == AtomicExpr::AO__atomic_load ||
952 E->getOp() == AtomicExpr::AO__atomic_load_n;
955 llvm::IntegerType::get(getLLVMContext(), Size * 8);
956 llvm::Value *OrigDest = GetDest();
957 Ptr = Builder.CreateBitCast(
958 Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
959 if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
960 if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
961 if (Dest && !E->isCmpXChg())
962 Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
964 if (isa<llvm::ConstantInt>(Order)) {
965 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
967 case AtomicExpr::AO_ABI_memory_order_relaxed:
968 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
969 Size, Align, llvm::Monotonic);
971 case AtomicExpr::AO_ABI_memory_order_consume:
972 case AtomicExpr::AO_ABI_memory_order_acquire:
974 break; // Avoid crashing on code with undefined behavior
975 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
976 Size, Align, llvm::Acquire);
978 case AtomicExpr::AO_ABI_memory_order_release:
980 break; // Avoid crashing on code with undefined behavior
981 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
982 Size, Align, llvm::Release);
984 case AtomicExpr::AO_ABI_memory_order_acq_rel:
985 if (IsLoad || IsStore)
986 break; // Avoid crashing on code with undefined behavior
987 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
988 Size, Align, llvm::AcquireRelease);
990 case AtomicExpr::AO_ABI_memory_order_seq_cst:
991 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
992 Size, Align, llvm::SequentiallyConsistent);
994 default: // invalid order
995 // We should not ever get here normally, but it's hard to
996 // enforce that in general.
999 if (RValTy->isVoidType())
1000 return RValue::get(nullptr);
1001 return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1004 // Long case, when Order isn't obviously constant.
1006 // Create all the relevant BB's
1007 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1008 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1009 *SeqCstBB = nullptr;
1010 MonotonicBB = createBasicBlock("monotonic", CurFn);
1012 AcquireBB = createBasicBlock("acquire", CurFn);
1014 ReleaseBB = createBasicBlock("release", CurFn);
1015 if (!IsLoad && !IsStore)
1016 AcqRelBB = createBasicBlock("acqrel", CurFn);
1017 SeqCstBB = createBasicBlock("seqcst", CurFn);
1018 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1020 // Create the switch for the split
1021 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1022 // doesn't matter unless someone is crazy enough to use something that
1023 // doesn't fold to a constant for the ordering.
1024 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1025 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1027 // Emit all the different atomics
1028 Builder.SetInsertPoint(MonotonicBB);
1029 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1030 Size, Align, llvm::Monotonic);
1031 Builder.CreateBr(ContBB);
1033 Builder.SetInsertPoint(AcquireBB);
1034 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1035 Size, Align, llvm::Acquire);
1036 Builder.CreateBr(ContBB);
1037 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1039 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1043 Builder.SetInsertPoint(ReleaseBB);
1044 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1045 Size, Align, llvm::Release);
1046 Builder.CreateBr(ContBB);
1047 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1050 if (!IsLoad && !IsStore) {
1051 Builder.SetInsertPoint(AcqRelBB);
1052 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1053 Size, Align, llvm::AcquireRelease);
1054 Builder.CreateBr(ContBB);
1055 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1058 Builder.SetInsertPoint(SeqCstBB);
1059 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1060 Size, Align, llvm::SequentiallyConsistent);
1061 Builder.CreateBr(ContBB);
1062 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1065 // Cleanup and return
1066 Builder.SetInsertPoint(ContBB);
1067 if (RValTy->isVoidType())
1068 return RValue::get(nullptr);
1069 return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1072 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
1073 unsigned addrspace =
1074 cast<llvm::PointerType>(addr->getType())->getAddressSpace();
1075 llvm::IntegerType *ty =
1076 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1077 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1080 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
1081 AggValueSlot resultSlot,
1082 SourceLocation loc, bool AsValue) const {
1083 if (LVal.isSimple()) {
1084 if (EvaluationKind == TEK_Aggregate)
1085 return resultSlot.asRValue();
1087 // Drill into the padding structure if we have one.
1089 addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
1091 // Otherwise, just convert the temporary to an r-value using the
1092 // normal conversion routine.
1093 return CGF.convertTempToRValue(addr, getValueType(), loc);
1096 // Get RValue from temp memory as atomic for non-simple lvalues
1098 CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
1099 if (LVal.isBitField())
1100 return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
1101 addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
1102 if (LVal.isVectorElt())
1103 return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
1105 LVal.getAlignment()),
1107 assert(LVal.isExtVectorElt());
1108 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1109 addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
1112 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1113 AggValueSlot ResultSlot,
1115 bool AsValue) const {
1116 // Try not to in some easy cases.
1117 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1118 if (getEvaluationKind() == TEK_Scalar &&
1119 (((!LVal.isBitField() ||
1120 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1123 auto *ValTy = AsValue
1124 ? CGF.ConvertTypeForMem(ValueTy)
1125 : getAtomicAddress()->getType()->getPointerElementType();
1126 if (ValTy->isIntegerTy()) {
1127 assert(IntVal->getType() == ValTy && "Different integer types.");
1128 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1129 } else if (ValTy->isPointerTy())
1130 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1131 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1132 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1135 // Create a temporary. This needs to be big enough to hold the
1138 bool TempIsVolatile = false;
1139 CharUnits TempAlignment;
1140 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1141 assert(!ResultSlot.isIgnored());
1142 Temp = ResultSlot.getAddr();
1143 TempAlignment = getValueAlignment();
1144 TempIsVolatile = ResultSlot.isVolatile();
1146 Temp = CreateTempAlloca();
1147 TempAlignment = getAtomicAlignment();
1150 // Slam the integer into the temporary.
1151 llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
1152 CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
1153 ->setVolatile(TempIsVolatile);
1155 return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
1158 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1159 llvm::AtomicOrdering AO, bool) {
1160 // void __atomic_load(size_t size, void *mem, void *return, int order);
1162 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1163 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1164 CGF.getContext().VoidPtrTy);
1165 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1166 CGF.getContext().VoidPtrTy);
1167 Args.add(RValue::get(
1168 llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1169 CGF.getContext().IntTy);
1170 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1173 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1175 // Okay, we're doing this natively.
1176 llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1177 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1178 Load->setAtomic(AO);
1180 // Other decoration.
1181 Load->setAlignment(getAtomicAlignment().getQuantity());
1183 Load->setVolatile(true);
1184 if (LVal.getTBAAInfo())
1185 CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
1189 /// An LValue is a candidate for having its loads and stores be made atomic if
1190 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1191 /// performing such an operation can be performed without a libcall.
1192 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1193 AtomicInfo AI(*this, LV);
1194 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1195 // An atomic is inline if we don't need to use a libcall.
1196 bool AtomicIsInline = !AI.shouldUseLibcall();
1197 return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1200 /// An type is a candidate for having its loads and stores be made atomic if
1201 /// we are operating under /volatile:ms *and* we know the access is volatile and
1202 /// performing such an operation can be performed without a libcall.
1203 bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
1204 bool IsVolatile) const {
1205 // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1206 bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1207 getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1208 return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1211 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1212 AggValueSlot Slot) {
1213 llvm::AtomicOrdering AO;
1214 bool IsVolatile = LV.isVolatileQualified();
1215 if (LV.getType()->isAtomicType()) {
1216 AO = llvm::SequentiallyConsistent;
1221 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1224 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1225 bool AsValue, llvm::AtomicOrdering AO,
1227 // Check whether we should use a library call.
1228 if (shouldUseLibcall()) {
1229 llvm::Value *TempAddr;
1230 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1231 assert(getEvaluationKind() == TEK_Aggregate);
1232 TempAddr = ResultSlot.getAddr();
1234 TempAddr = CreateTempAlloca();
1236 EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
1238 // Okay, turn that back into the original value or whole atomic (for
1239 // non-simple lvalues) type.
1240 return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1243 // Okay, we're doing this natively.
1244 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1246 // If we're ignoring an aggregate return, don't do anything.
1247 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1248 return RValue::getAggregate(nullptr, false);
1250 // Okay, turn that back into the original value or atomic (for non-simple
1252 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1255 /// Emit a load from an l-value of atomic type. Note that the r-value
1256 /// we produce is an r-value of the atomic *value* type.
1257 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1258 llvm::AtomicOrdering AO, bool IsVolatile,
1259 AggValueSlot resultSlot) {
1260 AtomicInfo Atomics(*this, src);
1261 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1265 /// Copy an r-value into memory as part of storing to an atomic type.
1266 /// This needs to create a bit-pattern suitable for atomic operations.
1267 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1268 assert(LVal.isSimple());
1269 // If we have an r-value, the rvalue should be of the atomic type,
1270 // which means that the caller is responsible for having zeroed
1271 // any padding. Just do an aggregate copy of that type.
1272 if (rvalue.isAggregate()) {
1273 CGF.EmitAggregateCopy(getAtomicAddress(),
1274 rvalue.getAggregateAddr(),
1276 (rvalue.isVolatileQualified()
1277 || LVal.isVolatileQualified()),
1278 LVal.getAlignment());
1282 // Okay, otherwise we're copying stuff.
1284 // Zero out the buffer if necessary.
1285 emitMemSetZeroIfNecessary();
1287 // Drill past the padding if present.
1288 LValue TempLVal = projectValue();
1290 // Okay, store the rvalue in.
1291 if (rvalue.isScalar()) {
1292 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1294 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1299 /// Materialize an r-value into memory for the purposes of storing it
1300 /// to an atomic type.
1301 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1302 // Aggregate r-values are already in memory, and EmitAtomicStore
1303 // requires them to be values of the atomic type.
1304 if (rvalue.isAggregate())
1305 return rvalue.getAggregateAddr();
1307 // Otherwise, make a temporary and materialize into it.
1308 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
1309 getAtomicAlignment());
1310 AtomicInfo Atomics(CGF, TempLV);
1311 Atomics.emitCopyIntoMemory(rvalue);
1312 return TempLV.getAddress();
1315 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1316 // If we've got a scalar value of the right size, try to avoid going
1318 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1319 llvm::Value *Value = RVal.getScalarVal();
1320 if (isa<llvm::IntegerType>(Value->getType()))
1321 return CGF.EmitToMemory(Value, ValueTy);
1323 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1324 CGF.getLLVMContext(),
1325 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1326 if (isa<llvm::PointerType>(Value->getType()))
1327 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1328 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1329 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1332 // Otherwise, we need to go through memory.
1333 // Put the r-value in memory.
1334 llvm::Value *Addr = materializeRValue(RVal);
1336 // Cast the temporary to the atomic int type and pull a value out.
1337 Addr = emitCastToAtomicIntPointer(Addr);
1338 return CGF.Builder.CreateAlignedLoad(Addr,
1339 getAtomicAlignment().getQuantity());
1342 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1343 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1344 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1345 // Do the atomic store.
1346 auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1347 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
1349 // Other decoration.
1350 Inst->setVolatile(LVal.isVolatileQualified());
1351 Inst->setWeak(IsWeak);
1353 // Okay, turn that back into the original value type.
1354 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1355 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1356 return std::make_pair(PreviousVal, SuccessFailureVal);
1360 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1361 llvm::Value *DesiredAddr,
1362 llvm::AtomicOrdering Success,
1363 llvm::AtomicOrdering Failure) {
1364 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1365 // void *desired, int success, int failure);
1367 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1368 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1369 CGF.getContext().VoidPtrTy);
1370 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1371 CGF.getContext().VoidPtrTy);
1372 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1373 CGF.getContext().VoidPtrTy);
1374 Args.add(RValue::get(llvm::ConstantInt::get(
1375 CGF.IntTy, translateAtomicOrdering(Success))),
1376 CGF.getContext().IntTy);
1377 Args.add(RValue::get(llvm::ConstantInt::get(
1378 CGF.IntTy, translateAtomicOrdering(Failure))),
1379 CGF.getContext().IntTy);
1380 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1381 CGF.getContext().BoolTy, Args);
1383 return SuccessFailureRVal.getScalarVal();
1386 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1387 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1388 llvm::AtomicOrdering Failure, bool IsWeak) {
1389 if (Failure >= Success)
1390 // Don't assert on undefined behavior.
1391 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1393 // Check whether we should use a library call.
1394 if (shouldUseLibcall()) {
1395 // Produce a source address.
1396 auto *ExpectedAddr = materializeRValue(Expected);
1397 auto *DesiredAddr = materializeRValue(Desired);
1398 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr,
1400 return std::make_pair(
1401 convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1402 SourceLocation(), /*AsValue=*/false),
1406 // If we've got a scalar value of the right size, try to avoid going
1408 auto *ExpectedVal = convertRValueToInt(Expected);
1409 auto *DesiredVal = convertRValueToInt(Desired);
1410 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1412 return std::make_pair(
1413 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1414 SourceLocation(), /*AsValue=*/false),
1419 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1420 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1421 llvm::Value *DesiredAddr) {
1422 llvm::Value *Ptr = nullptr;
1425 LValue AtomicLVal = Atomics.getAtomicLValue();
1427 if (AtomicLVal.isSimple()) {
1430 LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(),
1431 AtomicLVal.getAlignment(), CGF.CGM.getContext());
1433 // Build new lvalue for temp address
1434 Ptr = Atomics.materializeRValue(OldRVal);
1435 if (AtomicLVal.isBitField()) {
1437 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1438 AtomicLVal.getType(), AtomicLVal.getAlignment());
1440 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1441 AtomicLVal.getType(), AtomicLVal.getAlignment());
1442 } else if (AtomicLVal.isVectorElt()) {
1443 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1444 AtomicLVal.getType(),
1445 AtomicLVal.getAlignment());
1446 DesiredLVal = LValue::MakeVectorElt(
1447 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1448 AtomicLVal.getAlignment());
1450 assert(AtomicLVal.isExtVectorElt());
1451 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1452 AtomicLVal.getType(),
1453 AtomicLVal.getAlignment());
1454 DesiredLVal = LValue::MakeExtVectorElt(
1455 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1456 AtomicLVal.getAlignment());
1458 UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1459 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1460 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1462 // Store new value in the corresponding memory area
1463 RValue NewRVal = UpdateOp(UpRVal);
1464 if (NewRVal.isScalar()) {
1465 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1467 assert(NewRVal.isComplex());
1468 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1473 void AtomicInfo::EmitAtomicUpdateLibcall(
1474 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1476 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1478 llvm::Value *ExpectedAddr = CreateTempAlloca();
1480 EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
1481 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1482 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1483 CGF.EmitBlock(ContBB);
1484 auto *DesiredAddr = CreateTempAlloca();
1485 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1487 getAtomicAddress()->getType()->getPointerElementType())) {
1488 auto *OldVal = CGF.Builder.CreateAlignedLoad(
1489 ExpectedAddr, getAtomicAlignment().getQuantity());
1490 CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
1491 getAtomicAlignment().getQuantity());
1493 auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1494 SourceLocation(), /*AsValue=*/false);
1495 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1497 EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
1498 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1499 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1502 void AtomicInfo::EmitAtomicUpdateOp(
1503 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1505 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1507 // Do the atomic load.
1508 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1509 // For non-simple lvalues perform compare-and-swap procedure.
1510 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1511 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1512 auto *CurBB = CGF.Builder.GetInsertBlock();
1513 CGF.EmitBlock(ContBB);
1514 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1515 /*NumReservedValues=*/2);
1516 PHI->addIncoming(OldVal, CurBB);
1517 auto *NewAtomicAddr = CreateTempAlloca();
1518 auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1519 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1521 getAtomicAddress()->getType()->getPointerElementType())) {
1522 CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
1523 getAtomicAlignment().getQuantity());
1525 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1526 SourceLocation(), /*AsValue=*/false);
1527 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1528 auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
1529 NewAtomicIntAddr, getAtomicAlignment().getQuantity());
1530 // Try to write new value using cmpxchg operation
1531 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1532 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1533 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1534 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1537 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1538 RValue UpdateRVal, llvm::Value *DesiredAddr) {
1539 LValue AtomicLVal = Atomics.getAtomicLValue();
1541 // Build new lvalue for temp address
1542 if (AtomicLVal.isBitField()) {
1544 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1545 AtomicLVal.getType(), AtomicLVal.getAlignment());
1546 } else if (AtomicLVal.isVectorElt()) {
1548 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1549 AtomicLVal.getType(), AtomicLVal.getAlignment());
1551 assert(AtomicLVal.isExtVectorElt());
1552 DesiredLVal = LValue::MakeExtVectorElt(
1553 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1554 AtomicLVal.getAlignment());
1556 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1557 // Store new value in the corresponding memory area
1558 assert(UpdateRVal.isScalar());
1559 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1562 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1563 RValue UpdateRVal, bool IsVolatile) {
1564 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1566 llvm::Value *ExpectedAddr = CreateTempAlloca();
1568 EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
1569 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1570 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1571 CGF.EmitBlock(ContBB);
1572 auto *DesiredAddr = CreateTempAlloca();
1573 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1575 getAtomicAddress()->getType()->getPointerElementType())) {
1576 auto *OldVal = CGF.Builder.CreateAlignedLoad(
1577 ExpectedAddr, getAtomicAlignment().getQuantity());
1578 CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
1579 getAtomicAlignment().getQuantity());
1581 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1583 EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
1584 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1585 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1588 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1590 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1592 // Do the atomic load.
1593 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1594 // For non-simple lvalues perform compare-and-swap procedure.
1595 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1596 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1597 auto *CurBB = CGF.Builder.GetInsertBlock();
1598 CGF.EmitBlock(ContBB);
1599 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1600 /*NumReservedValues=*/2);
1601 PHI->addIncoming(OldVal, CurBB);
1602 auto *NewAtomicAddr = CreateTempAlloca();
1603 auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1604 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1606 getAtomicAddress()->getType()->getPointerElementType())) {
1607 CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
1608 getAtomicAlignment().getQuantity());
1610 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1611 auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
1612 NewAtomicIntAddr, getAtomicAlignment().getQuantity());
1613 // Try to write new value using cmpxchg operation
1614 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1615 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1616 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1617 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1620 void AtomicInfo::EmitAtomicUpdate(
1621 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1623 if (shouldUseLibcall()) {
1624 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1626 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1630 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1632 if (shouldUseLibcall()) {
1633 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1635 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1639 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1641 bool IsVolatile = lvalue.isVolatileQualified();
1642 llvm::AtomicOrdering AO;
1643 if (lvalue.getType()->isAtomicType()) {
1644 AO = llvm::SequentiallyConsistent;
1649 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1652 /// Emit a store to an l-value of atomic type.
1654 /// Note that the r-value is expected to be an r-value *of the atomic
1655 /// type*; this means that for aggregate r-values, it should include
1656 /// storage for any padding that was necessary.
1657 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1658 llvm::AtomicOrdering AO, bool IsVolatile,
1660 // If this is an aggregate r-value, it should agree in type except
1661 // maybe for address-space qualification.
1662 assert(!rvalue.isAggregate() ||
1663 rvalue.getAggregateAddr()->getType()->getPointerElementType()
1664 == dest.getAddress()->getType()->getPointerElementType());
1666 AtomicInfo atomics(*this, dest);
1667 LValue LVal = atomics.getAtomicLValue();
1669 // If this is an initialization, just put the value there normally.
1670 if (LVal.isSimple()) {
1672 atomics.emitCopyIntoMemory(rvalue);
1676 // Check whether we should use a library call.
1677 if (atomics.shouldUseLibcall()) {
1678 // Produce a source address.
1679 llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1681 // void __atomic_store(size_t size, void *mem, void *val, int order)
1683 args.add(RValue::get(atomics.getAtomicSizeValue()),
1684 getContext().getSizeType());
1685 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
1686 getContext().VoidPtrTy);
1687 args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
1688 args.add(RValue::get(llvm::ConstantInt::get(
1689 IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1690 getContext().IntTy);
1691 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1695 // Okay, we're doing this natively.
1696 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1698 // Do the atomic store.
1700 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1701 intValue = Builder.CreateIntCast(
1702 intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
1703 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1705 // Initializations don't need to be atomic.
1707 store->setAtomic(AO);
1709 // Other decoration.
1710 store->setAlignment(dest.getAlignment().getQuantity());
1712 store->setVolatile(true);
1713 if (dest.getTBAAInfo())
1714 CGM.DecorateInstruction(store, dest.getTBAAInfo());
1718 // Emit simple atomic update operation.
1719 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1722 /// Emit a compare-and-exchange op for atomic type.
1724 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1725 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1726 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1727 AggValueSlot Slot) {
1728 // If this is an aggregate r-value, it should agree in type except
1729 // maybe for address-space qualification.
1730 assert(!Expected.isAggregate() ||
1731 Expected.getAggregateAddr()->getType()->getPointerElementType() ==
1732 Obj.getAddress()->getType()->getPointerElementType());
1733 assert(!Desired.isAggregate() ||
1734 Desired.getAggregateAddr()->getType()->getPointerElementType() ==
1735 Obj.getAddress()->getType()->getPointerElementType());
1736 AtomicInfo Atomics(*this, Obj);
1738 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1742 void CodeGenFunction::EmitAtomicUpdate(
1743 LValue LVal, llvm::AtomicOrdering AO,
1744 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1745 AtomicInfo Atomics(*this, LVal);
1746 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1749 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1750 AtomicInfo atomics(*this, dest);
1752 switch (atomics.getEvaluationKind()) {
1754 llvm::Value *value = EmitScalarExpr(init);
1755 atomics.emitCopyIntoMemory(RValue::get(value));
1760 ComplexPairTy value = EmitComplexExpr(init);
1761 atomics.emitCopyIntoMemory(RValue::getComplex(value));
1765 case TEK_Aggregate: {
1766 // Fix up the destination if the initializer isn't an expression
1768 bool Zeroed = false;
1769 if (!init->getType()->isAtomicType()) {
1770 Zeroed = atomics.emitMemSetZeroIfNecessary();
1771 dest = atomics.projectValue();
1774 // Evaluate the expression directly into the destination.
1775 AggValueSlot slot = AggValueSlot::forLValue(dest,
1776 AggValueSlot::IsNotDestructed,
1777 AggValueSlot::DoesNotNeedGCBarriers,
1778 AggValueSlot::IsNotAliased,
1779 Zeroed ? AggValueSlot::IsZeroed :
1780 AggValueSlot::IsNotZeroed);
1782 EmitAggExpr(init, slot);
1786 llvm_unreachable("bad evaluation kind");