1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the code for emitting atomic operations.
12 //===----------------------------------------------------------------------===//
14 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "clang/AST/ASTContext.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/IR/DataLayout.h"
20 #include "llvm/IR/Intrinsics.h"
21 #include "llvm/IR/Operator.h"
23 using namespace clang;
24 using namespace CodeGen;
26 // The ABI values for various atomic memory orderings.
27 enum AtomicOrderingKind {
28 AO_ABI_memory_order_relaxed = 0,
29 AO_ABI_memory_order_consume = 1,
30 AO_ABI_memory_order_acquire = 2,
31 AO_ABI_memory_order_release = 3,
32 AO_ABI_memory_order_acq_rel = 4,
33 AO_ABI_memory_order_seq_cst = 5
41 uint64_t AtomicSizeInBits;
42 uint64_t ValueSizeInBits;
43 CharUnits AtomicAlign;
45 CharUnits LValueAlign;
46 TypeEvaluationKind EvaluationKind;
49 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) {
50 assert(lvalue.isSimple());
52 AtomicTy = lvalue.getType();
53 ValueTy = AtomicTy->castAs<AtomicType>()->getValueType();
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
56 ASTContext &C = CGF.getContext();
58 uint64_t valueAlignInBits;
59 llvm::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy);
61 uint64_t atomicAlignInBits;
62 llvm::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy);
64 assert(ValueSizeInBits <= AtomicSizeInBits);
65 assert(valueAlignInBits <= atomicAlignInBits);
67 AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits);
68 ValueAlign = C.toCharUnitsFromBits(valueAlignInBits);
69 if (lvalue.getAlignment().isZero())
70 lvalue.setAlignment(AtomicAlign);
73 (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) ||
74 AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth());
77 QualType getAtomicType() const { return AtomicTy; }
78 QualType getValueType() const { return ValueTy; }
79 CharUnits getAtomicAlignment() const { return AtomicAlign; }
80 CharUnits getValueAlignment() const { return ValueAlign; }
81 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
82 uint64_t getValueSizeInBits() const { return AtomicSizeInBits; }
83 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
84 bool shouldUseLibcall() const { return UseLibcall; }
86 /// Is the atomic size larger than the underlying value type?
88 /// Note that the absence of padding does not mean that atomic
89 /// objects are completely interchangeable with non-atomic
90 /// objects: we might have promoted the alignment of a type
91 /// without making it bigger.
92 bool hasPadding() const {
93 return (ValueSizeInBits != AtomicSizeInBits);
96 void emitMemSetZeroIfNecessary(LValue dest) const;
98 llvm::Value *getAtomicSizeValue() const {
99 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
100 return CGF.CGM.getSize(size);
103 /// Cast the given pointer to an integer pointer suitable for
104 /// atomic operations.
105 llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
107 /// Turn an atomic-layout object into an r-value.
108 RValue convertTempToRValue(llvm::Value *addr,
109 AggValueSlot resultSlot) const;
111 /// Copy an atomic r-value into atomic-layout memory.
112 void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const;
114 /// Project an l-value down to the value field.
115 LValue projectValue(LValue lvalue) const {
116 llvm::Value *addr = lvalue.getAddress();
118 addr = CGF.Builder.CreateStructGEP(addr, 0);
120 return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(),
121 CGF.getContext(), lvalue.getTBAAInfo());
124 /// Materialize an atomic r-value in atomic-layout memory.
125 llvm::Value *materializeRValue(RValue rvalue) const;
128 bool requiresMemSetZero(llvm::Type *type) const;
132 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
136 const CGFunctionInfo &fnInfo =
137 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
138 FunctionType::ExtInfo(), RequiredArgs::All);
139 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
140 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
141 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
144 /// Does a store of the given IR type modify the full expected width?
145 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
146 uint64_t expectedSize) {
147 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
150 /// Does the atomic type require memsetting to zero before initialization?
152 /// The IR type is provided as a way of making certain queries faster.
153 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
154 // If the atomic type has size padding, we definitely need a memset.
155 if (hasPadding()) return true;
157 // Otherwise, do some simple heuristics to try to avoid it:
158 switch (getEvaluationKind()) {
159 // For scalars and complexes, check whether the store size of the
160 // type uses the full size.
162 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
164 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
165 AtomicSizeInBits / 2);
167 // Just be pessimistic about aggregates.
171 llvm_unreachable("bad evaluation kind");
174 void AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
175 llvm::Value *addr = dest.getAddress();
176 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
179 CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
180 AtomicSizeInBits / 8,
181 dest.getAlignment().getQuantity());
185 EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
186 llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
187 uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
188 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
189 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
191 switch (E->getOp()) {
192 case AtomicExpr::AO__c11_atomic_init:
193 llvm_unreachable("Already handled!");
195 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
196 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
197 case AtomicExpr::AO__atomic_compare_exchange:
198 case AtomicExpr::AO__atomic_compare_exchange_n: {
199 // Note that cmpxchg only supports specifying one ordering and
200 // doesn't support weak cmpxchg, at least at the moment.
201 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
202 LoadVal1->setAlignment(Align);
203 llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
204 LoadVal2->setAlignment(Align);
205 llvm::AtomicCmpXchgInst *CXI =
206 CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
207 CXI->setVolatile(E->isVolatile());
208 llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
209 StoreVal1->setAlignment(Align);
210 llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
211 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
215 case AtomicExpr::AO__c11_atomic_load:
216 case AtomicExpr::AO__atomic_load_n:
217 case AtomicExpr::AO__atomic_load: {
218 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
219 Load->setAtomic(Order);
220 Load->setAlignment(Size);
221 Load->setVolatile(E->isVolatile());
222 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
223 StoreDest->setAlignment(Align);
227 case AtomicExpr::AO__c11_atomic_store:
228 case AtomicExpr::AO__atomic_store:
229 case AtomicExpr::AO__atomic_store_n: {
230 assert(!Dest && "Store does not return a value");
231 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
232 LoadVal1->setAlignment(Align);
233 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
234 Store->setAtomic(Order);
235 Store->setAlignment(Size);
236 Store->setVolatile(E->isVolatile());
240 case AtomicExpr::AO__c11_atomic_exchange:
241 case AtomicExpr::AO__atomic_exchange_n:
242 case AtomicExpr::AO__atomic_exchange:
243 Op = llvm::AtomicRMWInst::Xchg;
246 case AtomicExpr::AO__atomic_add_fetch:
247 PostOp = llvm::Instruction::Add;
249 case AtomicExpr::AO__c11_atomic_fetch_add:
250 case AtomicExpr::AO__atomic_fetch_add:
251 Op = llvm::AtomicRMWInst::Add;
254 case AtomicExpr::AO__atomic_sub_fetch:
255 PostOp = llvm::Instruction::Sub;
257 case AtomicExpr::AO__c11_atomic_fetch_sub:
258 case AtomicExpr::AO__atomic_fetch_sub:
259 Op = llvm::AtomicRMWInst::Sub;
262 case AtomicExpr::AO__atomic_and_fetch:
263 PostOp = llvm::Instruction::And;
265 case AtomicExpr::AO__c11_atomic_fetch_and:
266 case AtomicExpr::AO__atomic_fetch_and:
267 Op = llvm::AtomicRMWInst::And;
270 case AtomicExpr::AO__atomic_or_fetch:
271 PostOp = llvm::Instruction::Or;
273 case AtomicExpr::AO__c11_atomic_fetch_or:
274 case AtomicExpr::AO__atomic_fetch_or:
275 Op = llvm::AtomicRMWInst::Or;
278 case AtomicExpr::AO__atomic_xor_fetch:
279 PostOp = llvm::Instruction::Xor;
281 case AtomicExpr::AO__c11_atomic_fetch_xor:
282 case AtomicExpr::AO__atomic_fetch_xor:
283 Op = llvm::AtomicRMWInst::Xor;
286 case AtomicExpr::AO__atomic_nand_fetch:
287 PostOp = llvm::Instruction::And;
289 case AtomicExpr::AO__atomic_fetch_nand:
290 Op = llvm::AtomicRMWInst::Nand;
294 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
295 LoadVal1->setAlignment(Align);
296 llvm::AtomicRMWInst *RMWI =
297 CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
298 RMWI->setVolatile(E->isVolatile());
300 // For __atomic_*_fetch operations, perform the operation again to
301 // determine the value which was written.
302 llvm::Value *Result = RMWI;
304 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
305 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
306 Result = CGF.Builder.CreateNot(Result);
307 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
308 StoreDest->setAlignment(Align);
311 // This function emits any expression (scalar, complex, or aggregate)
312 // into a temporary alloca.
314 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
315 llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
316 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
322 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
323 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy) {
324 if (UseOptimizedLibcall) {
325 // Load value and pass it to the function directly.
326 unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
327 Val = CGF.EmitLoadOfScalar(Val, false, Align, ValTy);
328 Args.add(RValue::get(Val), ValTy);
330 // Non-optimized functions always take a reference.
331 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
332 CGF.getContext().VoidPtrTy);
336 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
337 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
338 QualType MemTy = AtomicTy;
339 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
340 MemTy = AT->getValueType();
341 CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
342 uint64_t Size = sizeChars.getQuantity();
343 CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
344 unsigned Align = alignChars.getQuantity();
345 unsigned MaxInlineWidthInBits =
346 getTarget().getMaxAtomicInlineWidth();
347 bool UseLibcall = (Size != Align ||
348 getContext().toBits(sizeChars) > MaxInlineWidthInBits);
350 llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
351 Ptr = EmitScalarExpr(E->getPtr());
353 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
354 assert(!Dest && "Init does not return a value");
355 LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
356 EmitAtomicInit(E->getVal1(), lvalue);
357 return RValue::get(0);
360 Order = EmitScalarExpr(E->getOrder());
362 switch (E->getOp()) {
363 case AtomicExpr::AO__c11_atomic_init:
364 llvm_unreachable("Already handled!");
366 case AtomicExpr::AO__c11_atomic_load:
367 case AtomicExpr::AO__atomic_load_n:
370 case AtomicExpr::AO__atomic_load:
371 Dest = EmitScalarExpr(E->getVal1());
374 case AtomicExpr::AO__atomic_store:
375 Val1 = EmitScalarExpr(E->getVal1());
378 case AtomicExpr::AO__atomic_exchange:
379 Val1 = EmitScalarExpr(E->getVal1());
380 Dest = EmitScalarExpr(E->getVal2());
383 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
384 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
385 case AtomicExpr::AO__atomic_compare_exchange_n:
386 case AtomicExpr::AO__atomic_compare_exchange:
387 Val1 = EmitScalarExpr(E->getVal1());
388 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
389 Val2 = EmitScalarExpr(E->getVal2());
391 Val2 = EmitValToTemp(*this, E->getVal2());
392 OrderFail = EmitScalarExpr(E->getOrderFail());
393 // Evaluate and discard the 'weak' argument.
394 if (E->getNumSubExprs() == 6)
395 EmitScalarExpr(E->getWeak());
398 case AtomicExpr::AO__c11_atomic_fetch_add:
399 case AtomicExpr::AO__c11_atomic_fetch_sub:
400 if (MemTy->isPointerType()) {
401 // For pointer arithmetic, we're required to do a bit of math:
402 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
403 // ... but only for the C11 builtins. The GNU builtins expect the
404 // user to multiply by sizeof(T).
405 QualType Val1Ty = E->getVal1()->getType();
406 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
407 CharUnits PointeeIncAmt =
408 getContext().getTypeSizeInChars(MemTy->getPointeeType());
409 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
410 Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
411 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
415 case AtomicExpr::AO__atomic_fetch_add:
416 case AtomicExpr::AO__atomic_fetch_sub:
417 case AtomicExpr::AO__atomic_add_fetch:
418 case AtomicExpr::AO__atomic_sub_fetch:
419 case AtomicExpr::AO__c11_atomic_store:
420 case AtomicExpr::AO__c11_atomic_exchange:
421 case AtomicExpr::AO__atomic_store_n:
422 case AtomicExpr::AO__atomic_exchange_n:
423 case AtomicExpr::AO__c11_atomic_fetch_and:
424 case AtomicExpr::AO__c11_atomic_fetch_or:
425 case AtomicExpr::AO__c11_atomic_fetch_xor:
426 case AtomicExpr::AO__atomic_fetch_and:
427 case AtomicExpr::AO__atomic_fetch_or:
428 case AtomicExpr::AO__atomic_fetch_xor:
429 case AtomicExpr::AO__atomic_fetch_nand:
430 case AtomicExpr::AO__atomic_and_fetch:
431 case AtomicExpr::AO__atomic_or_fetch:
432 case AtomicExpr::AO__atomic_xor_fetch:
433 case AtomicExpr::AO__atomic_nand_fetch:
434 Val1 = EmitValToTemp(*this, E->getVal1());
438 if (!E->getType()->isVoidType() && !Dest)
439 Dest = CreateMemTemp(E->getType(), ".atomicdst");
441 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
443 bool UseOptimizedLibcall = false;
444 switch (E->getOp()) {
445 case AtomicExpr::AO__c11_atomic_fetch_add:
446 case AtomicExpr::AO__atomic_fetch_add:
447 case AtomicExpr::AO__c11_atomic_fetch_and:
448 case AtomicExpr::AO__atomic_fetch_and:
449 case AtomicExpr::AO__c11_atomic_fetch_or:
450 case AtomicExpr::AO__atomic_fetch_or:
451 case AtomicExpr::AO__c11_atomic_fetch_sub:
452 case AtomicExpr::AO__atomic_fetch_sub:
453 case AtomicExpr::AO__c11_atomic_fetch_xor:
454 case AtomicExpr::AO__atomic_fetch_xor:
455 // For these, only library calls for certain sizes exist.
456 UseOptimizedLibcall = true;
459 // Only use optimized library calls for sizes for which they exist.
460 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
461 UseOptimizedLibcall = true;
466 if (!UseOptimizedLibcall) {
467 // For non-optimized library calls, the size is the first parameter
468 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
469 getContext().getSizeType());
471 // Atomic address is the first or second parameter
472 Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
473 getContext().VoidPtrTy);
475 std::string LibCallName;
477 bool HaveRetTy = false;
478 switch (E->getOp()) {
479 // There is only one libcall for compare an exchange, because there is no
480 // optimisation benefit possible from a libcall version of a weak compare
482 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
483 // void *desired, int success, int failure)
484 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
485 // int success, int failure)
486 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
487 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
488 case AtomicExpr::AO__atomic_compare_exchange:
489 case AtomicExpr::AO__atomic_compare_exchange_n:
490 LibCallName = "__atomic_compare_exchange";
491 RetTy = getContext().BoolTy;
493 Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
494 getContext().VoidPtrTy);
495 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy);
496 Args.add(RValue::get(Order),
500 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
502 // T __atomic_exchange_N(T *mem, T val, int order)
503 case AtomicExpr::AO__c11_atomic_exchange:
504 case AtomicExpr::AO__atomic_exchange_n:
505 case AtomicExpr::AO__atomic_exchange:
506 LibCallName = "__atomic_exchange";
507 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
509 // void __atomic_store(size_t size, void *mem, void *val, int order)
510 // void __atomic_store_N(T *mem, T val, int order)
511 case AtomicExpr::AO__c11_atomic_store:
512 case AtomicExpr::AO__atomic_store:
513 case AtomicExpr::AO__atomic_store_n:
514 LibCallName = "__atomic_store";
515 RetTy = getContext().VoidTy;
517 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
519 // void __atomic_load(size_t size, void *mem, void *return, int order)
520 // T __atomic_load_N(T *mem, int order)
521 case AtomicExpr::AO__c11_atomic_load:
522 case AtomicExpr::AO__atomic_load:
523 case AtomicExpr::AO__atomic_load_n:
524 LibCallName = "__atomic_load";
526 // T __atomic_fetch_add_N(T *mem, T val, int order)
527 case AtomicExpr::AO__c11_atomic_fetch_add:
528 case AtomicExpr::AO__atomic_fetch_add:
529 LibCallName = "__atomic_fetch_add";
530 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
532 // T __atomic_fetch_and_N(T *mem, T val, int order)
533 case AtomicExpr::AO__c11_atomic_fetch_and:
534 case AtomicExpr::AO__atomic_fetch_and:
535 LibCallName = "__atomic_fetch_and";
536 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
538 // T __atomic_fetch_or_N(T *mem, T val, int order)
539 case AtomicExpr::AO__c11_atomic_fetch_or:
540 case AtomicExpr::AO__atomic_fetch_or:
541 LibCallName = "__atomic_fetch_or";
542 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
544 // T __atomic_fetch_sub_N(T *mem, T val, int order)
545 case AtomicExpr::AO__c11_atomic_fetch_sub:
546 case AtomicExpr::AO__atomic_fetch_sub:
547 LibCallName = "__atomic_fetch_sub";
548 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
550 // T __atomic_fetch_xor_N(T *mem, T val, int order)
551 case AtomicExpr::AO__c11_atomic_fetch_xor:
552 case AtomicExpr::AO__atomic_fetch_xor:
553 LibCallName = "__atomic_fetch_xor";
554 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy);
556 default: return EmitUnsupportedRValue(E, "atomic library call");
559 // Optimized functions have the size in their name.
560 if (UseOptimizedLibcall)
561 LibCallName += "_" + llvm::utostr(Size);
562 // By default, assume we return a value of the atomic type.
564 if (UseOptimizedLibcall) {
565 // Value is returned directly.
568 // Value is returned through parameter before the order.
569 RetTy = getContext().VoidTy;
570 Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
571 getContext().VoidPtrTy);
574 // order is always the last parameter
575 Args.add(RValue::get(Order),
578 const CGFunctionInfo &FuncInfo =
579 CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args,
580 FunctionType::ExtInfo(), RequiredArgs::All);
581 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
582 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
583 RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
584 if (!RetTy->isVoidType())
586 if (E->getType()->isVoidType())
587 return RValue::get(0);
588 return convertTempToRValue(Dest, E->getType());
591 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
592 E->getOp() == AtomicExpr::AO__atomic_store ||
593 E->getOp() == AtomicExpr::AO__atomic_store_n;
594 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
595 E->getOp() == AtomicExpr::AO__atomic_load ||
596 E->getOp() == AtomicExpr::AO__atomic_load_n;
599 llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
600 llvm::Value *OrigDest = Dest;
601 Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
602 if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
603 if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
604 if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
606 if (isa<llvm::ConstantInt>(Order)) {
607 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
609 case AO_ABI_memory_order_relaxed:
610 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
613 case AO_ABI_memory_order_consume:
614 case AO_ABI_memory_order_acquire:
616 break; // Avoid crashing on code with undefined behavior
617 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
620 case AO_ABI_memory_order_release:
622 break; // Avoid crashing on code with undefined behavior
623 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
626 case AO_ABI_memory_order_acq_rel:
627 if (IsLoad || IsStore)
628 break; // Avoid crashing on code with undefined behavior
629 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
630 llvm::AcquireRelease);
632 case AO_ABI_memory_order_seq_cst:
633 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
634 llvm::SequentiallyConsistent);
636 default: // invalid order
637 // We should not ever get here normally, but it's hard to
638 // enforce that in general.
641 if (E->getType()->isVoidType())
642 return RValue::get(0);
643 return convertTempToRValue(OrigDest, E->getType());
646 // Long case, when Order isn't obviously constant.
648 // Create all the relevant BB's
649 llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
650 *AcqRelBB = 0, *SeqCstBB = 0;
651 MonotonicBB = createBasicBlock("monotonic", CurFn);
653 AcquireBB = createBasicBlock("acquire", CurFn);
655 ReleaseBB = createBasicBlock("release", CurFn);
656 if (!IsLoad && !IsStore)
657 AcqRelBB = createBasicBlock("acqrel", CurFn);
658 SeqCstBB = createBasicBlock("seqcst", CurFn);
659 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
661 // Create the switch for the split
662 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
663 // doesn't matter unless someone is crazy enough to use something that
664 // doesn't fold to a constant for the ordering.
665 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
666 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
668 // Emit all the different atomics
669 Builder.SetInsertPoint(MonotonicBB);
670 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
672 Builder.CreateBr(ContBB);
674 Builder.SetInsertPoint(AcquireBB);
675 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
677 Builder.CreateBr(ContBB);
678 SI->addCase(Builder.getInt32(1), AcquireBB);
679 SI->addCase(Builder.getInt32(2), AcquireBB);
682 Builder.SetInsertPoint(ReleaseBB);
683 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
685 Builder.CreateBr(ContBB);
686 SI->addCase(Builder.getInt32(3), ReleaseBB);
688 if (!IsLoad && !IsStore) {
689 Builder.SetInsertPoint(AcqRelBB);
690 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
691 llvm::AcquireRelease);
692 Builder.CreateBr(ContBB);
693 SI->addCase(Builder.getInt32(4), AcqRelBB);
695 Builder.SetInsertPoint(SeqCstBB);
696 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
697 llvm::SequentiallyConsistent);
698 Builder.CreateBr(ContBB);
699 SI->addCase(Builder.getInt32(5), SeqCstBB);
701 // Cleanup and return
702 Builder.SetInsertPoint(ContBB);
703 if (E->getType()->isVoidType())
704 return RValue::get(0);
705 return convertTempToRValue(OrigDest, E->getType());
708 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
710 cast<llvm::PointerType>(addr->getType())->getAddressSpace();
711 llvm::IntegerType *ty =
712 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
713 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
716 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
717 AggValueSlot resultSlot) const {
718 if (EvaluationKind == TEK_Aggregate) {
719 // Nothing to do if the result is ignored.
720 if (resultSlot.isIgnored()) return resultSlot.asRValue();
722 assert(resultSlot.getAddr() == addr || hasPadding());
724 // In these cases, we should have emitted directly into the result slot.
725 if (!hasPadding() || resultSlot.isValueOfAtomic())
726 return resultSlot.asRValue();
728 // Otherwise, fall into the common path.
731 // Drill into the padding structure if we have one.
733 addr = CGF.Builder.CreateStructGEP(addr, 0);
735 // If we're emitting to an aggregate, copy into the result slot.
736 if (EvaluationKind == TEK_Aggregate) {
737 CGF.EmitAggregateCopy(resultSlot.getAddr(), addr, getValueType(),
738 resultSlot.isVolatile());
739 return resultSlot.asRValue();
742 // Otherwise, just convert the temporary to an r-value using the
743 // normal conversion routine.
744 return CGF.convertTempToRValue(addr, getValueType());
747 /// Emit a load from an l-value of atomic type. Note that the r-value
748 /// we produce is an r-value of the atomic *value* type.
749 RValue CodeGenFunction::EmitAtomicLoad(LValue src, AggValueSlot resultSlot) {
750 AtomicInfo atomics(*this, src);
752 // Check whether we should use a library call.
753 if (atomics.shouldUseLibcall()) {
754 llvm::Value *tempAddr;
755 if (resultSlot.isValueOfAtomic()) {
756 assert(atomics.getEvaluationKind() == TEK_Aggregate);
757 tempAddr = resultSlot.getPaddedAtomicAddr();
758 } else if (!resultSlot.isIgnored() && !atomics.hasPadding()) {
759 assert(atomics.getEvaluationKind() == TEK_Aggregate);
760 tempAddr = resultSlot.getAddr();
762 tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
765 // void __atomic_load(size_t size, void *mem, void *return, int order);
767 args.add(RValue::get(atomics.getAtomicSizeValue()),
768 getContext().getSizeType());
769 args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
770 getContext().VoidPtrTy);
771 args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
772 getContext().VoidPtrTy);
773 args.add(RValue::get(llvm::ConstantInt::get(IntTy,
774 AO_ABI_memory_order_seq_cst)),
776 emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
778 // Produce the r-value.
779 return atomics.convertTempToRValue(tempAddr, resultSlot);
782 // Okay, we're doing this natively.
783 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
784 llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
785 load->setAtomic(llvm::SequentiallyConsistent);
788 load->setAlignment(src.getAlignment().getQuantity());
789 if (src.isVolatileQualified())
790 load->setVolatile(true);
791 if (src.getTBAAInfo())
792 CGM.DecorateInstruction(load, src.getTBAAInfo());
794 // Okay, turn that back into the original value type.
795 QualType valueType = atomics.getValueType();
796 llvm::Value *result = load;
798 // If we're ignoring an aggregate return, don't do anything.
799 if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
800 return RValue::getAggregate(0, false);
802 // The easiest way to do this this is to go through memory, but we
803 // try not to in some easy cases.
804 if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) {
805 llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
806 if (isa<llvm::IntegerType>(resultTy)) {
807 assert(result->getType() == resultTy);
808 result = EmitFromMemory(result, valueType);
809 } else if (isa<llvm::PointerType>(resultTy)) {
810 result = Builder.CreateIntToPtr(result, resultTy);
812 result = Builder.CreateBitCast(result, resultTy);
814 return RValue::get(result);
817 // Create a temporary. This needs to be big enough to hold the
820 bool tempIsVolatile = false;
821 CharUnits tempAlignment;
822 if (atomics.getEvaluationKind() == TEK_Aggregate &&
823 (!atomics.hasPadding() || resultSlot.isValueOfAtomic())) {
824 assert(!resultSlot.isIgnored());
825 if (resultSlot.isValueOfAtomic()) {
826 temp = resultSlot.getPaddedAtomicAddr();
827 tempAlignment = atomics.getAtomicAlignment();
829 temp = resultSlot.getAddr();
830 tempAlignment = atomics.getValueAlignment();
832 tempIsVolatile = resultSlot.isVolatile();
834 temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
835 tempAlignment = atomics.getAtomicAlignment();
838 // Slam the integer into the temporary.
839 llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
840 Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity())
841 ->setVolatile(tempIsVolatile);
843 return atomics.convertTempToRValue(temp, resultSlot);
848 /// Copy an r-value into memory as part of storing to an atomic type.
849 /// This needs to create a bit-pattern suitable for atomic operations.
850 void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
851 // If we have an r-value, the rvalue should be of the atomic type,
852 // which means that the caller is responsible for having zeroed
853 // any padding. Just do an aggregate copy of that type.
854 if (rvalue.isAggregate()) {
855 CGF.EmitAggregateCopy(dest.getAddress(),
856 rvalue.getAggregateAddr(),
858 (rvalue.isVolatileQualified()
859 || dest.isVolatileQualified()),
860 dest.getAlignment());
864 // Okay, otherwise we're copying stuff.
866 // Zero out the buffer if necessary.
867 emitMemSetZeroIfNecessary(dest);
869 // Drill past the padding if present.
870 dest = projectValue(dest);
872 // Okay, store the rvalue in.
873 if (rvalue.isScalar()) {
874 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
876 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
881 /// Materialize an r-value into memory for the purposes of storing it
882 /// to an atomic type.
883 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
884 // Aggregate r-values are already in memory, and EmitAtomicStore
885 // requires them to be values of the atomic type.
886 if (rvalue.isAggregate())
887 return rvalue.getAggregateAddr();
889 // Otherwise, make a temporary and materialize into it.
890 llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
891 LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment());
892 emitCopyIntoMemory(rvalue, tempLV);
896 /// Emit a store to an l-value of atomic type.
898 /// Note that the r-value is expected to be an r-value *of the atomic
899 /// type*; this means that for aggregate r-values, it should include
900 /// storage for any padding that was necessary.
901 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
903 // If this is an aggregate r-value, it should agree in type except
904 // maybe for address-space qualification.
905 assert(!rvalue.isAggregate() ||
906 rvalue.getAggregateAddr()->getType()->getPointerElementType()
907 == dest.getAddress()->getType()->getPointerElementType());
909 AtomicInfo atomics(*this, dest);
911 // If this is an initialization, just put the value there normally.
913 atomics.emitCopyIntoMemory(rvalue, dest);
917 // Check whether we should use a library call.
918 if (atomics.shouldUseLibcall()) {
919 // Produce a source address.
920 llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
922 // void __atomic_store(size_t size, void *mem, void *val, int order)
924 args.add(RValue::get(atomics.getAtomicSizeValue()),
925 getContext().getSizeType());
926 args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
927 getContext().VoidPtrTy);
928 args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
929 getContext().VoidPtrTy);
930 args.add(RValue::get(llvm::ConstantInt::get(IntTy,
931 AO_ABI_memory_order_seq_cst)),
933 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
937 // Okay, we're doing this natively.
938 llvm::Value *intValue;
940 // If we've got a scalar value of the right size, try to avoid going
942 if (rvalue.isScalar() && !atomics.hasPadding()) {
943 llvm::Value *value = rvalue.getScalarVal();
944 if (isa<llvm::IntegerType>(value->getType())) {
947 llvm::IntegerType *inputIntTy =
948 llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits());
949 if (isa<llvm::PointerType>(value->getType())) {
950 intValue = Builder.CreatePtrToInt(value, inputIntTy);
952 intValue = Builder.CreateBitCast(value, inputIntTy);
956 // Otherwise, we need to go through memory.
958 // Put the r-value in memory.
959 llvm::Value *addr = atomics.materializeRValue(rvalue);
961 // Cast the temporary to the atomic int type and pull a value out.
962 addr = atomics.emitCastToAtomicIntPointer(addr);
963 intValue = Builder.CreateAlignedLoad(addr,
964 atomics.getAtomicAlignment().getQuantity());
967 // Do the atomic store.
968 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
969 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
971 // Initializations don't need to be atomic.
972 if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
975 store->setAlignment(dest.getAlignment().getQuantity());
976 if (dest.isVolatileQualified())
977 store->setVolatile(true);
978 if (dest.getTBAAInfo())
979 CGM.DecorateInstruction(store, dest.getTBAAInfo());
982 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
983 AtomicInfo atomics(*this, dest);
985 switch (atomics.getEvaluationKind()) {
987 llvm::Value *value = EmitScalarExpr(init);
988 atomics.emitCopyIntoMemory(RValue::get(value), dest);
993 ComplexPairTy value = EmitComplexExpr(init);
994 atomics.emitCopyIntoMemory(RValue::getComplex(value), dest);
998 case TEK_Aggregate: {
999 // Memset the buffer first if there's any possibility of
1000 // uninitialized internal bits.
1001 atomics.emitMemSetZeroIfNecessary(dest);
1003 // HACK: whether the initializer actually has an atomic type
1004 // doesn't really seem reliable right now.
1005 if (!init->getType()->isAtomicType()) {
1006 dest = atomics.projectValue(dest);
1009 // Evaluate the expression directly into the destination.
1010 AggValueSlot slot = AggValueSlot::forLValue(dest,
1011 AggValueSlot::IsNotDestructed,
1012 AggValueSlot::DoesNotNeedGCBarriers,
1013 AggValueSlot::IsNotAliased);
1014 EmitAggExpr(init, slot);
1018 llvm_unreachable("bad evaluation kind");