1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the code for emitting atomic operations.
12 //===----------------------------------------------------------------------===//
14 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "clang/AST/ASTContext.h"
18 #include "clang/CodeGen/CGFunctionInfo.h"
19 #include "llvm/ADT/StringExtras.h"
20 #include "llvm/IR/DataLayout.h"
21 #include "llvm/IR/Intrinsics.h"
22 #include "llvm/IR/Operator.h"
24 using namespace clang;
25 using namespace CodeGen;
32 uint64_t AtomicSizeInBits;
33 uint64_t ValueSizeInBits;
34 CharUnits AtomicAlign;
36 CharUnits LValueAlign;
37 TypeEvaluationKind EvaluationKind;
40 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) {
41 assert(lvalue.isSimple());
43 AtomicTy = lvalue.getType();
44 ValueTy = AtomicTy->castAs<AtomicType>()->getValueType();
45 EvaluationKind = CGF.getEvaluationKind(ValueTy);
47 ASTContext &C = CGF.getContext();
49 uint64_t ValueAlignInBits;
50 uint64_t AtomicAlignInBits;
51 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
52 ValueSizeInBits = ValueTI.Width;
53 ValueAlignInBits = ValueTI.Align;
55 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
56 AtomicSizeInBits = AtomicTI.Width;
57 AtomicAlignInBits = AtomicTI.Align;
59 assert(ValueSizeInBits <= AtomicSizeInBits);
60 assert(ValueAlignInBits <= AtomicAlignInBits);
62 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
63 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
64 if (lvalue.getAlignment().isZero())
65 lvalue.setAlignment(AtomicAlign);
67 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
68 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
71 QualType getAtomicType() const { return AtomicTy; }
72 QualType getValueType() const { return ValueTy; }
73 CharUnits getAtomicAlignment() const { return AtomicAlign; }
74 CharUnits getValueAlignment() const { return ValueAlign; }
75 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
76 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
77 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
78 bool shouldUseLibcall() const { return UseLibcall; }
80 /// Is the atomic size larger than the underlying value type?
82 /// Note that the absence of padding does not mean that atomic
83 /// objects are completely interchangeable with non-atomic
84 /// objects: we might have promoted the alignment of a type
85 /// without making it bigger.
86 bool hasPadding() const {
87 return (ValueSizeInBits != AtomicSizeInBits);
90 bool emitMemSetZeroIfNecessary(LValue dest) const;
92 llvm::Value *getAtomicSizeValue() const {
93 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
94 return CGF.CGM.getSize(size);
97 /// Cast the given pointer to an integer pointer suitable for
98 /// atomic operations.
99 llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
101 /// Turn an atomic-layout object into an r-value.
102 RValue convertTempToRValue(llvm::Value *addr,
103 AggValueSlot resultSlot,
104 SourceLocation loc) const;
106 /// \brief Converts a rvalue to integer value.
107 llvm::Value *convertRValueToInt(RValue RVal) const;
109 RValue convertIntToValue(llvm::Value *IntVal, AggValueSlot ResultSlot,
110 SourceLocation Loc) const;
112 /// Copy an atomic r-value into atomic-layout memory.
113 void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const;
115 /// Project an l-value down to the value field.
116 LValue projectValue(LValue lvalue) const {
117 llvm::Value *addr = lvalue.getAddress();
119 addr = CGF.Builder.CreateStructGEP(addr, 0);
121 return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(),
122 CGF.getContext(), lvalue.getTBAAInfo());
125 /// Materialize an atomic r-value in atomic-layout memory.
126 llvm::Value *materializeRValue(RValue rvalue) const;
129 bool requiresMemSetZero(llvm::Type *type) const;
133 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
137 const CGFunctionInfo &fnInfo =
138 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
139 FunctionType::ExtInfo(), RequiredArgs::All);
140 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
141 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
142 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
145 /// Does a store of the given IR type modify the full expected width?
146 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
147 uint64_t expectedSize) {
148 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
151 /// Does the atomic type require memsetting to zero before initialization?
153 /// The IR type is provided as a way of making certain queries faster.
154 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
155 // If the atomic type has size padding, we definitely need a memset.
156 if (hasPadding()) return true;
158 // Otherwise, do some simple heuristics to try to avoid it:
159 switch (getEvaluationKind()) {
160 // For scalars and complexes, check whether the store size of the
161 // type uses the full size.
163 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
165 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
166 AtomicSizeInBits / 2);
168 // Padding in structs has an undefined bit pattern. User beware.
172 llvm_unreachable("bad evaluation kind");
175 bool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
176 llvm::Value *addr = dest.getAddress();
177 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
180 CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
181 AtomicSizeInBits / 8,
182 dest.getAlignment().getQuantity());
186 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
187 llvm::Value *Dest, llvm::Value *Ptr,
188 llvm::Value *Val1, llvm::Value *Val2,
189 uint64_t Size, unsigned Align,
190 llvm::AtomicOrdering SuccessOrder,
191 llvm::AtomicOrdering FailureOrder) {
192 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
193 llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
194 Expected->setAlignment(Align);
195 llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
196 Desired->setAlignment(Align);
198 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
199 Ptr, Expected, Desired, SuccessOrder, FailureOrder);
200 Pair->setVolatile(E->isVolatile());
201 Pair->setWeak(IsWeak);
203 // Cmp holds the result of the compare-exchange operation: true on success,
205 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
206 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
208 // This basic block is used to hold the store instruction if the operation
210 llvm::BasicBlock *StoreExpectedBB =
211 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
213 // This basic block is the exit point of the operation, we should end up
214 // here regardless of whether or not the operation succeeded.
215 llvm::BasicBlock *ContinueBB =
216 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
218 // Update Expected if Expected isn't equal to Old, otherwise branch to the
220 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
222 CGF.Builder.SetInsertPoint(StoreExpectedBB);
223 // Update the memory at Expected with Old's value.
224 llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
225 StoreExpected->setAlignment(Align);
226 // Finally, branch to the exit point.
227 CGF.Builder.CreateBr(ContinueBB);
229 CGF.Builder.SetInsertPoint(ContinueBB);
230 // Update the memory at Dest with Cmp's value.
231 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
235 /// Given an ordering required on success, emit all possible cmpxchg
236 /// instructions to cope with the provided (but possibly only dynamically known)
238 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
239 bool IsWeak, llvm::Value *Dest,
240 llvm::Value *Ptr, llvm::Value *Val1,
242 llvm::Value *FailureOrderVal,
243 uint64_t Size, unsigned Align,
244 llvm::AtomicOrdering SuccessOrder) {
245 llvm::AtomicOrdering FailureOrder;
246 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
247 switch (FO->getSExtValue()) {
249 FailureOrder = llvm::Monotonic;
251 case AtomicExpr::AO_ABI_memory_order_consume:
252 case AtomicExpr::AO_ABI_memory_order_acquire:
253 FailureOrder = llvm::Acquire;
255 case AtomicExpr::AO_ABI_memory_order_seq_cst:
256 FailureOrder = llvm::SequentiallyConsistent;
259 if (FailureOrder >= SuccessOrder) {
260 // Don't assert on undefined behaviour.
262 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
264 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
265 SuccessOrder, FailureOrder);
269 // Create all the relevant BB's
270 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
272 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
273 if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
274 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
275 if (SuccessOrder == llvm::SequentiallyConsistent)
276 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
278 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
280 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
282 // Emit all the different atomics
284 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
285 // doesn't matter unless someone is crazy enough to use something that
286 // doesn't fold to a constant for the ordering.
287 CGF.Builder.SetInsertPoint(MonotonicBB);
288 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
289 Size, Align, SuccessOrder, llvm::Monotonic);
290 CGF.Builder.CreateBr(ContBB);
293 CGF.Builder.SetInsertPoint(AcquireBB);
294 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
295 Size, Align, SuccessOrder, llvm::Acquire);
296 CGF.Builder.CreateBr(ContBB);
297 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
299 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
303 CGF.Builder.SetInsertPoint(SeqCstBB);
304 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
305 Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
306 CGF.Builder.CreateBr(ContBB);
307 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
311 CGF.Builder.SetInsertPoint(ContBB);
314 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
315 llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
316 llvm::Value *IsWeak, llvm::Value *FailureOrder,
317 uint64_t Size, unsigned Align,
318 llvm::AtomicOrdering Order) {
319 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
320 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
322 switch (E->getOp()) {
323 case AtomicExpr::AO__c11_atomic_init:
324 llvm_unreachable("Already handled!");
326 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
327 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
328 FailureOrder, Size, Align, Order);
330 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
331 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
332 FailureOrder, Size, Align, Order);
334 case AtomicExpr::AO__atomic_compare_exchange:
335 case AtomicExpr::AO__atomic_compare_exchange_n: {
336 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
337 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
338 Val1, Val2, FailureOrder, Size, Align, Order);
340 // Create all the relevant BB's
341 llvm::BasicBlock *StrongBB =
342 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
343 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
344 llvm::BasicBlock *ContBB =
345 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
347 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
348 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
350 CGF.Builder.SetInsertPoint(StrongBB);
351 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
352 FailureOrder, Size, Align, Order);
353 CGF.Builder.CreateBr(ContBB);
355 CGF.Builder.SetInsertPoint(WeakBB);
356 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
357 FailureOrder, Size, Align, Order);
358 CGF.Builder.CreateBr(ContBB);
360 CGF.Builder.SetInsertPoint(ContBB);
364 case AtomicExpr::AO__c11_atomic_load:
365 case AtomicExpr::AO__atomic_load_n:
366 case AtomicExpr::AO__atomic_load: {
367 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
368 Load->setAtomic(Order);
369 Load->setAlignment(Size);
370 Load->setVolatile(E->isVolatile());
371 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
372 StoreDest->setAlignment(Align);
376 case AtomicExpr::AO__c11_atomic_store:
377 case AtomicExpr::AO__atomic_store:
378 case AtomicExpr::AO__atomic_store_n: {
379 assert(!Dest && "Store does not return a value");
380 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
381 LoadVal1->setAlignment(Align);
382 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
383 Store->setAtomic(Order);
384 Store->setAlignment(Size);
385 Store->setVolatile(E->isVolatile());
389 case AtomicExpr::AO__c11_atomic_exchange:
390 case AtomicExpr::AO__atomic_exchange_n:
391 case AtomicExpr::AO__atomic_exchange:
392 Op = llvm::AtomicRMWInst::Xchg;
395 case AtomicExpr::AO__atomic_add_fetch:
396 PostOp = llvm::Instruction::Add;
398 case AtomicExpr::AO__c11_atomic_fetch_add:
399 case AtomicExpr::AO__atomic_fetch_add:
400 Op = llvm::AtomicRMWInst::Add;
403 case AtomicExpr::AO__atomic_sub_fetch:
404 PostOp = llvm::Instruction::Sub;
406 case AtomicExpr::AO__c11_atomic_fetch_sub:
407 case AtomicExpr::AO__atomic_fetch_sub:
408 Op = llvm::AtomicRMWInst::Sub;
411 case AtomicExpr::AO__atomic_and_fetch:
412 PostOp = llvm::Instruction::And;
414 case AtomicExpr::AO__c11_atomic_fetch_and:
415 case AtomicExpr::AO__atomic_fetch_and:
416 Op = llvm::AtomicRMWInst::And;
419 case AtomicExpr::AO__atomic_or_fetch:
420 PostOp = llvm::Instruction::Or;
422 case AtomicExpr::AO__c11_atomic_fetch_or:
423 case AtomicExpr::AO__atomic_fetch_or:
424 Op = llvm::AtomicRMWInst::Or;
427 case AtomicExpr::AO__atomic_xor_fetch:
428 PostOp = llvm::Instruction::Xor;
430 case AtomicExpr::AO__c11_atomic_fetch_xor:
431 case AtomicExpr::AO__atomic_fetch_xor:
432 Op = llvm::AtomicRMWInst::Xor;
435 case AtomicExpr::AO__atomic_nand_fetch:
436 PostOp = llvm::Instruction::And;
438 case AtomicExpr::AO__atomic_fetch_nand:
439 Op = llvm::AtomicRMWInst::Nand;
443 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
444 LoadVal1->setAlignment(Align);
445 llvm::AtomicRMWInst *RMWI =
446 CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
447 RMWI->setVolatile(E->isVolatile());
449 // For __atomic_*_fetch operations, perform the operation again to
450 // determine the value which was written.
451 llvm::Value *Result = RMWI;
453 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
454 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
455 Result = CGF.Builder.CreateNot(Result);
456 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
457 StoreDest->setAlignment(Align);
460 // This function emits any expression (scalar, complex, or aggregate)
461 // into a temporary alloca.
463 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
464 llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
465 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
471 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
472 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
473 SourceLocation Loc, CharUnits SizeInChars) {
474 if (UseOptimizedLibcall) {
475 // Load value and pass it to the function directly.
476 unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
477 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
479 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
480 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
481 SizeInBits)->getPointerTo();
482 Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
483 Align, CGF.getContext().getPointerType(ValTy),
485 // Coerce the value into an appropriately sized integer type.
486 Args.add(RValue::get(Val), ValTy);
488 // Non-optimized functions always take a reference.
489 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
490 CGF.getContext().VoidPtrTy);
494 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
495 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
496 QualType MemTy = AtomicTy;
497 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
498 MemTy = AT->getValueType();
499 CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
500 uint64_t Size = sizeChars.getQuantity();
501 CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
502 unsigned Align = alignChars.getQuantity();
503 unsigned MaxInlineWidthInBits =
504 getTarget().getMaxAtomicInlineWidth();
505 bool UseLibcall = (Size != Align ||
506 getContext().toBits(sizeChars) > MaxInlineWidthInBits);
508 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
510 llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
512 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
513 assert(!Dest && "Init does not return a value");
514 LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
515 EmitAtomicInit(E->getVal1(), lvalue);
516 return RValue::get(nullptr);
519 llvm::Value *Order = EmitScalarExpr(E->getOrder());
521 switch (E->getOp()) {
522 case AtomicExpr::AO__c11_atomic_init:
523 llvm_unreachable("Already handled!");
525 case AtomicExpr::AO__c11_atomic_load:
526 case AtomicExpr::AO__atomic_load_n:
529 case AtomicExpr::AO__atomic_load:
530 Dest = EmitScalarExpr(E->getVal1());
533 case AtomicExpr::AO__atomic_store:
534 Val1 = EmitScalarExpr(E->getVal1());
537 case AtomicExpr::AO__atomic_exchange:
538 Val1 = EmitScalarExpr(E->getVal1());
539 Dest = EmitScalarExpr(E->getVal2());
542 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
543 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
544 case AtomicExpr::AO__atomic_compare_exchange_n:
545 case AtomicExpr::AO__atomic_compare_exchange:
546 Val1 = EmitScalarExpr(E->getVal1());
547 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
548 Val2 = EmitScalarExpr(E->getVal2());
550 Val2 = EmitValToTemp(*this, E->getVal2());
551 OrderFail = EmitScalarExpr(E->getOrderFail());
552 if (E->getNumSubExprs() == 6)
553 IsWeak = EmitScalarExpr(E->getWeak());
556 case AtomicExpr::AO__c11_atomic_fetch_add:
557 case AtomicExpr::AO__c11_atomic_fetch_sub:
558 if (MemTy->isPointerType()) {
559 // For pointer arithmetic, we're required to do a bit of math:
560 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
561 // ... but only for the C11 builtins. The GNU builtins expect the
562 // user to multiply by sizeof(T).
563 QualType Val1Ty = E->getVal1()->getType();
564 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
565 CharUnits PointeeIncAmt =
566 getContext().getTypeSizeInChars(MemTy->getPointeeType());
567 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
568 Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
569 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
573 case AtomicExpr::AO__atomic_fetch_add:
574 case AtomicExpr::AO__atomic_fetch_sub:
575 case AtomicExpr::AO__atomic_add_fetch:
576 case AtomicExpr::AO__atomic_sub_fetch:
577 case AtomicExpr::AO__c11_atomic_store:
578 case AtomicExpr::AO__c11_atomic_exchange:
579 case AtomicExpr::AO__atomic_store_n:
580 case AtomicExpr::AO__atomic_exchange_n:
581 case AtomicExpr::AO__c11_atomic_fetch_and:
582 case AtomicExpr::AO__c11_atomic_fetch_or:
583 case AtomicExpr::AO__c11_atomic_fetch_xor:
584 case AtomicExpr::AO__atomic_fetch_and:
585 case AtomicExpr::AO__atomic_fetch_or:
586 case AtomicExpr::AO__atomic_fetch_xor:
587 case AtomicExpr::AO__atomic_fetch_nand:
588 case AtomicExpr::AO__atomic_and_fetch:
589 case AtomicExpr::AO__atomic_or_fetch:
590 case AtomicExpr::AO__atomic_xor_fetch:
591 case AtomicExpr::AO__atomic_nand_fetch:
592 Val1 = EmitValToTemp(*this, E->getVal1());
596 QualType RValTy = E->getType().getUnqualifiedType();
599 if (!RValTy->isVoidType() && !Dest) {
600 Dest = CreateMemTemp(RValTy, ".atomicdst");
605 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
607 bool UseOptimizedLibcall = false;
608 switch (E->getOp()) {
609 case AtomicExpr::AO__c11_atomic_fetch_add:
610 case AtomicExpr::AO__atomic_fetch_add:
611 case AtomicExpr::AO__c11_atomic_fetch_and:
612 case AtomicExpr::AO__atomic_fetch_and:
613 case AtomicExpr::AO__c11_atomic_fetch_or:
614 case AtomicExpr::AO__atomic_fetch_or:
615 case AtomicExpr::AO__c11_atomic_fetch_sub:
616 case AtomicExpr::AO__atomic_fetch_sub:
617 case AtomicExpr::AO__c11_atomic_fetch_xor:
618 case AtomicExpr::AO__atomic_fetch_xor:
619 // For these, only library calls for certain sizes exist.
620 UseOptimizedLibcall = true;
623 // Only use optimized library calls for sizes for which they exist.
624 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
625 UseOptimizedLibcall = true;
630 if (!UseOptimizedLibcall) {
631 // For non-optimized library calls, the size is the first parameter
632 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
633 getContext().getSizeType());
635 // Atomic address is the first or second parameter
636 Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
638 std::string LibCallName;
639 QualType LoweredMemTy =
640 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
642 bool HaveRetTy = false;
643 switch (E->getOp()) {
644 // There is only one libcall for compare an exchange, because there is no
645 // optimisation benefit possible from a libcall version of a weak compare
647 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
648 // void *desired, int success, int failure)
649 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
650 // int success, int failure)
651 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
652 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
653 case AtomicExpr::AO__atomic_compare_exchange:
654 case AtomicExpr::AO__atomic_compare_exchange_n:
655 LibCallName = "__atomic_compare_exchange";
656 RetTy = getContext().BoolTy;
658 Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
659 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
660 E->getExprLoc(), sizeChars);
661 Args.add(RValue::get(Order), getContext().IntTy);
664 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
666 // T __atomic_exchange_N(T *mem, T val, int order)
667 case AtomicExpr::AO__c11_atomic_exchange:
668 case AtomicExpr::AO__atomic_exchange_n:
669 case AtomicExpr::AO__atomic_exchange:
670 LibCallName = "__atomic_exchange";
671 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
672 E->getExprLoc(), sizeChars);
674 // void __atomic_store(size_t size, void *mem, void *val, int order)
675 // void __atomic_store_N(T *mem, T val, int order)
676 case AtomicExpr::AO__c11_atomic_store:
677 case AtomicExpr::AO__atomic_store:
678 case AtomicExpr::AO__atomic_store_n:
679 LibCallName = "__atomic_store";
680 RetTy = getContext().VoidTy;
682 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
683 E->getExprLoc(), sizeChars);
685 // void __atomic_load(size_t size, void *mem, void *return, int order)
686 // T __atomic_load_N(T *mem, int order)
687 case AtomicExpr::AO__c11_atomic_load:
688 case AtomicExpr::AO__atomic_load:
689 case AtomicExpr::AO__atomic_load_n:
690 LibCallName = "__atomic_load";
692 // T __atomic_fetch_add_N(T *mem, T val, int order)
693 case AtomicExpr::AO__c11_atomic_fetch_add:
694 case AtomicExpr::AO__atomic_fetch_add:
695 LibCallName = "__atomic_fetch_add";
696 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
697 E->getExprLoc(), sizeChars);
699 // T __atomic_fetch_and_N(T *mem, T val, int order)
700 case AtomicExpr::AO__c11_atomic_fetch_and:
701 case AtomicExpr::AO__atomic_fetch_and:
702 LibCallName = "__atomic_fetch_and";
703 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
704 E->getExprLoc(), sizeChars);
706 // T __atomic_fetch_or_N(T *mem, T val, int order)
707 case AtomicExpr::AO__c11_atomic_fetch_or:
708 case AtomicExpr::AO__atomic_fetch_or:
709 LibCallName = "__atomic_fetch_or";
710 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
711 E->getExprLoc(), sizeChars);
713 // T __atomic_fetch_sub_N(T *mem, T val, int order)
714 case AtomicExpr::AO__c11_atomic_fetch_sub:
715 case AtomicExpr::AO__atomic_fetch_sub:
716 LibCallName = "__atomic_fetch_sub";
717 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
718 E->getExprLoc(), sizeChars);
720 // T __atomic_fetch_xor_N(T *mem, T val, int order)
721 case AtomicExpr::AO__c11_atomic_fetch_xor:
722 case AtomicExpr::AO__atomic_fetch_xor:
723 LibCallName = "__atomic_fetch_xor";
724 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
725 E->getExprLoc(), sizeChars);
727 default: return EmitUnsupportedRValue(E, "atomic library call");
730 // Optimized functions have the size in their name.
731 if (UseOptimizedLibcall)
732 LibCallName += "_" + llvm::utostr(Size);
733 // By default, assume we return a value of the atomic type.
735 if (UseOptimizedLibcall) {
736 // Value is returned directly.
737 // The function returns an appropriately sized integer type.
738 RetTy = getContext().getIntTypeForBitwidth(
739 getContext().toBits(sizeChars), /*Signed=*/false);
741 // Value is returned through parameter before the order.
742 RetTy = getContext().VoidTy;
743 Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
746 // order is always the last parameter
747 Args.add(RValue::get(Order),
750 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
751 // The value is returned directly from the libcall.
752 if (HaveRetTy && !RetTy->isVoidType())
754 // The value is returned via an explicit out param.
755 if (RetTy->isVoidType())
756 return RValue::get(nullptr);
757 // The value is returned directly for optimized libcalls but the caller is
758 // expected an out-param.
759 if (UseOptimizedLibcall) {
760 llvm::Value *ResVal = Res.getScalarVal();
761 llvm::StoreInst *StoreDest = Builder.CreateStore(
763 Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
764 StoreDest->setAlignment(Align);
766 return convertTempToRValue(Dest, RValTy, E->getExprLoc());
769 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
770 E->getOp() == AtomicExpr::AO__atomic_store ||
771 E->getOp() == AtomicExpr::AO__atomic_store_n;
772 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
773 E->getOp() == AtomicExpr::AO__atomic_load ||
774 E->getOp() == AtomicExpr::AO__atomic_load_n;
777 llvm::IntegerType::get(getLLVMContext(), Size * 8);
778 llvm::Value *OrigDest = GetDest();
779 Ptr = Builder.CreateBitCast(
780 Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
781 if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
782 if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
783 if (Dest && !E->isCmpXChg())
784 Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
786 if (isa<llvm::ConstantInt>(Order)) {
787 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
789 case AtomicExpr::AO_ABI_memory_order_relaxed:
790 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
791 Size, Align, llvm::Monotonic);
793 case AtomicExpr::AO_ABI_memory_order_consume:
794 case AtomicExpr::AO_ABI_memory_order_acquire:
796 break; // Avoid crashing on code with undefined behavior
797 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
798 Size, Align, llvm::Acquire);
800 case AtomicExpr::AO_ABI_memory_order_release:
802 break; // Avoid crashing on code with undefined behavior
803 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
804 Size, Align, llvm::Release);
806 case AtomicExpr::AO_ABI_memory_order_acq_rel:
807 if (IsLoad || IsStore)
808 break; // Avoid crashing on code with undefined behavior
809 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
810 Size, Align, llvm::AcquireRelease);
812 case AtomicExpr::AO_ABI_memory_order_seq_cst:
813 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
814 Size, Align, llvm::SequentiallyConsistent);
816 default: // invalid order
817 // We should not ever get here normally, but it's hard to
818 // enforce that in general.
821 if (RValTy->isVoidType())
822 return RValue::get(nullptr);
823 return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
826 // Long case, when Order isn't obviously constant.
828 // Create all the relevant BB's
829 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
830 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
832 MonotonicBB = createBasicBlock("monotonic", CurFn);
834 AcquireBB = createBasicBlock("acquire", CurFn);
836 ReleaseBB = createBasicBlock("release", CurFn);
837 if (!IsLoad && !IsStore)
838 AcqRelBB = createBasicBlock("acqrel", CurFn);
839 SeqCstBB = createBasicBlock("seqcst", CurFn);
840 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
842 // Create the switch for the split
843 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
844 // doesn't matter unless someone is crazy enough to use something that
845 // doesn't fold to a constant for the ordering.
846 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
847 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
849 // Emit all the different atomics
850 Builder.SetInsertPoint(MonotonicBB);
851 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
852 Size, Align, llvm::Monotonic);
853 Builder.CreateBr(ContBB);
855 Builder.SetInsertPoint(AcquireBB);
856 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
857 Size, Align, llvm::Acquire);
858 Builder.CreateBr(ContBB);
859 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
861 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
865 Builder.SetInsertPoint(ReleaseBB);
866 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
867 Size, Align, llvm::Release);
868 Builder.CreateBr(ContBB);
869 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
872 if (!IsLoad && !IsStore) {
873 Builder.SetInsertPoint(AcqRelBB);
874 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
875 Size, Align, llvm::AcquireRelease);
876 Builder.CreateBr(ContBB);
877 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
880 Builder.SetInsertPoint(SeqCstBB);
881 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
882 Size, Align, llvm::SequentiallyConsistent);
883 Builder.CreateBr(ContBB);
884 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
887 // Cleanup and return
888 Builder.SetInsertPoint(ContBB);
889 if (RValTy->isVoidType())
890 return RValue::get(nullptr);
891 return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
894 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
896 cast<llvm::PointerType>(addr->getType())->getAddressSpace();
897 llvm::IntegerType *ty =
898 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
899 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
902 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
903 AggValueSlot resultSlot,
904 SourceLocation loc) const {
905 if (EvaluationKind == TEK_Aggregate)
906 return resultSlot.asRValue();
908 // Drill into the padding structure if we have one.
910 addr = CGF.Builder.CreateStructGEP(addr, 0);
912 // Otherwise, just convert the temporary to an r-value using the
913 // normal conversion routine.
914 return CGF.convertTempToRValue(addr, getValueType(), loc);
917 RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal,
918 AggValueSlot ResultSlot,
919 SourceLocation Loc) const {
920 // Try not to in some easy cases.
921 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
922 if (getEvaluationKind() == TEK_Scalar && !hasPadding()) {
923 auto *ValTy = CGF.ConvertTypeForMem(ValueTy);
924 if (ValTy->isIntegerTy()) {
925 assert(IntVal->getType() == ValTy && "Different integer types.");
926 return RValue::get(IntVal);
927 } else if (ValTy->isPointerTy())
928 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
929 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
930 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
933 // Create a temporary. This needs to be big enough to hold the
936 bool TempIsVolatile = false;
937 CharUnits TempAlignment;
938 if (getEvaluationKind() == TEK_Aggregate) {
939 assert(!ResultSlot.isIgnored());
940 Temp = ResultSlot.getAddr();
941 TempAlignment = getValueAlignment();
942 TempIsVolatile = ResultSlot.isVolatile();
944 Temp = CGF.CreateMemTemp(getAtomicType(), "atomic-temp");
945 TempAlignment = getAtomicAlignment();
948 // Slam the integer into the temporary.
949 llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
950 CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
951 ->setVolatile(TempIsVolatile);
953 return convertTempToRValue(Temp, ResultSlot, Loc);
956 /// Emit a load from an l-value of atomic type. Note that the r-value
957 /// we produce is an r-value of the atomic *value* type.
958 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
959 AggValueSlot resultSlot) {
960 AtomicInfo atomics(*this, src);
962 // Check whether we should use a library call.
963 if (atomics.shouldUseLibcall()) {
964 llvm::Value *tempAddr;
965 if (!resultSlot.isIgnored()) {
966 assert(atomics.getEvaluationKind() == TEK_Aggregate);
967 tempAddr = resultSlot.getAddr();
969 tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
972 // void __atomic_load(size_t size, void *mem, void *return, int order);
974 args.add(RValue::get(atomics.getAtomicSizeValue()),
975 getContext().getSizeType());
976 args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
977 getContext().VoidPtrTy);
978 args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
979 getContext().VoidPtrTy);
980 args.add(RValue::get(llvm::ConstantInt::get(
981 IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
983 emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
985 // Produce the r-value.
986 return atomics.convertTempToRValue(tempAddr, resultSlot, loc);
989 // Okay, we're doing this natively.
990 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
991 llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
992 load->setAtomic(llvm::SequentiallyConsistent);
995 load->setAlignment(src.getAlignment().getQuantity());
996 if (src.isVolatileQualified())
997 load->setVolatile(true);
998 if (src.getTBAAInfo())
999 CGM.DecorateInstruction(load, src.getTBAAInfo());
1001 // If we're ignoring an aggregate return, don't do anything.
1002 if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
1003 return RValue::getAggregate(nullptr, false);
1005 // Okay, turn that back into the original value type.
1006 return atomics.convertIntToValue(load, resultSlot, loc);
1011 /// Copy an r-value into memory as part of storing to an atomic type.
1012 /// This needs to create a bit-pattern suitable for atomic operations.
1013 void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
1014 // If we have an r-value, the rvalue should be of the atomic type,
1015 // which means that the caller is responsible for having zeroed
1016 // any padding. Just do an aggregate copy of that type.
1017 if (rvalue.isAggregate()) {
1018 CGF.EmitAggregateCopy(dest.getAddress(),
1019 rvalue.getAggregateAddr(),
1021 (rvalue.isVolatileQualified()
1022 || dest.isVolatileQualified()),
1023 dest.getAlignment());
1027 // Okay, otherwise we're copying stuff.
1029 // Zero out the buffer if necessary.
1030 emitMemSetZeroIfNecessary(dest);
1032 // Drill past the padding if present.
1033 dest = projectValue(dest);
1035 // Okay, store the rvalue in.
1036 if (rvalue.isScalar()) {
1037 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
1039 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
1044 /// Materialize an r-value into memory for the purposes of storing it
1045 /// to an atomic type.
1046 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1047 // Aggregate r-values are already in memory, and EmitAtomicStore
1048 // requires them to be values of the atomic type.
1049 if (rvalue.isAggregate())
1050 return rvalue.getAggregateAddr();
1052 // Otherwise, make a temporary and materialize into it.
1053 llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
1054 LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment());
1055 emitCopyIntoMemory(rvalue, tempLV);
1059 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1060 // If we've got a scalar value of the right size, try to avoid going
1062 if (RVal.isScalar() && !hasPadding()) {
1063 llvm::Value *Value = RVal.getScalarVal();
1064 if (isa<llvm::IntegerType>(Value->getType()))
1067 llvm::IntegerType *InputIntTy =
1068 llvm::IntegerType::get(CGF.getLLVMContext(), getValueSizeInBits());
1069 if (isa<llvm::PointerType>(Value->getType()))
1070 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1071 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1072 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1075 // Otherwise, we need to go through memory.
1076 // Put the r-value in memory.
1077 llvm::Value *Addr = materializeRValue(RVal);
1079 // Cast the temporary to the atomic int type and pull a value out.
1080 Addr = emitCastToAtomicIntPointer(Addr);
1081 return CGF.Builder.CreateAlignedLoad(Addr,
1082 getAtomicAlignment().getQuantity());
1085 /// Emit a store to an l-value of atomic type.
1087 /// Note that the r-value is expected to be an r-value *of the atomic
1088 /// type*; this means that for aggregate r-values, it should include
1089 /// storage for any padding that was necessary.
1090 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
1091 // If this is an aggregate r-value, it should agree in type except
1092 // maybe for address-space qualification.
1093 assert(!rvalue.isAggregate() ||
1094 rvalue.getAggregateAddr()->getType()->getPointerElementType()
1095 == dest.getAddress()->getType()->getPointerElementType());
1097 AtomicInfo atomics(*this, dest);
1099 // If this is an initialization, just put the value there normally.
1101 atomics.emitCopyIntoMemory(rvalue, dest);
1105 // Check whether we should use a library call.
1106 if (atomics.shouldUseLibcall()) {
1107 // Produce a source address.
1108 llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1110 // void __atomic_store(size_t size, void *mem, void *val, int order)
1112 args.add(RValue::get(atomics.getAtomicSizeValue()),
1113 getContext().getSizeType());
1114 args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
1115 getContext().VoidPtrTy);
1116 args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
1117 getContext().VoidPtrTy);
1118 args.add(RValue::get(llvm::ConstantInt::get(
1119 IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
1120 getContext().IntTy);
1121 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1125 // Okay, we're doing this natively.
1126 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1128 // Do the atomic store.
1129 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
1130 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1132 // Initializations don't need to be atomic.
1133 if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
1135 // Other decoration.
1136 store->setAlignment(dest.getAlignment().getQuantity());
1137 if (dest.isVolatileQualified())
1138 store->setVolatile(true);
1139 if (dest.getTBAAInfo())
1140 CGM.DecorateInstruction(store, dest.getTBAAInfo());
1143 /// Emit a compare-and-exchange op for atomic type.
1145 std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange(
1146 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1147 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1148 AggValueSlot Slot) {
1149 // If this is an aggregate r-value, it should agree in type except
1150 // maybe for address-space qualification.
1151 assert(!Expected.isAggregate() ||
1152 Expected.getAggregateAddr()->getType()->getPointerElementType() ==
1153 Obj.getAddress()->getType()->getPointerElementType());
1154 assert(!Desired.isAggregate() ||
1155 Desired.getAggregateAddr()->getType()->getPointerElementType() ==
1156 Obj.getAddress()->getType()->getPointerElementType());
1157 AtomicInfo Atomics(*this, Obj);
1159 if (Failure >= Success)
1160 // Don't assert on undefined behavior.
1161 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1163 auto Alignment = Atomics.getValueAlignment();
1164 // Check whether we should use a library call.
1165 if (Atomics.shouldUseLibcall()) {
1166 auto *ExpectedAddr = Atomics.materializeRValue(Expected);
1167 // Produce a source address.
1168 auto *DesiredAddr = Atomics.materializeRValue(Desired);
1169 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1170 // void *desired, int success, int failure);
1172 Args.add(RValue::get(Atomics.getAtomicSizeValue()),
1173 getContext().getSizeType());
1174 Args.add(RValue::get(EmitCastToVoidPtr(Obj.getAddress())),
1175 getContext().VoidPtrTy);
1176 Args.add(RValue::get(EmitCastToVoidPtr(ExpectedAddr)),
1177 getContext().VoidPtrTy);
1178 Args.add(RValue::get(EmitCastToVoidPtr(DesiredAddr)),
1179 getContext().VoidPtrTy);
1180 Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Success)),
1181 getContext().IntTy);
1182 Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Failure)),
1183 getContext().IntTy);
1184 auto SuccessFailureRVal = emitAtomicLibcall(
1185 *this, "__atomic_compare_exchange", getContext().BoolTy, Args);
1187 Builder.CreateAlignedLoad(ExpectedAddr, Alignment.getQuantity());
1188 return std::make_pair(RValue::get(PreviousVal), SuccessFailureRVal);
1191 // If we've got a scalar value of the right size, try to avoid going
1193 auto *ExpectedIntVal = Atomics.convertRValueToInt(Expected);
1194 auto *DesiredIntVal = Atomics.convertRValueToInt(Desired);
1196 // Do the atomic store.
1197 auto *Addr = Atomics.emitCastToAtomicIntPointer(Obj.getAddress());
1198 auto *Inst = Builder.CreateAtomicCmpXchg(Addr, ExpectedIntVal, DesiredIntVal,
1200 // Other decoration.
1201 Inst->setVolatile(Obj.isVolatileQualified());
1202 Inst->setWeak(IsWeak);
1204 // Okay, turn that back into the original value type.
1205 auto *PreviousVal = Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1206 auto *SuccessFailureVal = Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1207 return std::make_pair(Atomics.convertIntToValue(PreviousVal, Slot, Loc),
1208 RValue::get(SuccessFailureVal));
1211 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1212 AtomicInfo atomics(*this, dest);
1214 switch (atomics.getEvaluationKind()) {
1216 llvm::Value *value = EmitScalarExpr(init);
1217 atomics.emitCopyIntoMemory(RValue::get(value), dest);
1222 ComplexPairTy value = EmitComplexExpr(init);
1223 atomics.emitCopyIntoMemory(RValue::getComplex(value), dest);
1227 case TEK_Aggregate: {
1228 // Fix up the destination if the initializer isn't an expression
1230 bool Zeroed = false;
1231 if (!init->getType()->isAtomicType()) {
1232 Zeroed = atomics.emitMemSetZeroIfNecessary(dest);
1233 dest = atomics.projectValue(dest);
1236 // Evaluate the expression directly into the destination.
1237 AggValueSlot slot = AggValueSlot::forLValue(dest,
1238 AggValueSlot::IsNotDestructed,
1239 AggValueSlot::DoesNotNeedGCBarriers,
1240 AggValueSlot::IsNotAliased,
1241 Zeroed ? AggValueSlot::IsZeroed :
1242 AggValueSlot::IsNotZeroed);
1244 EmitAggExpr(init, slot);
1248 llvm_unreachable("bad evaluation kind");