1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This contains code to emit OpenMP nodes as LLVM code.
12 //===----------------------------------------------------------------------===//
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31 for (const auto *C : S.clauses()) {
32 if (auto *CPI = OMPClauseWithPreInit::get(C)) {
33 if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34 for (const auto *I : PreInit->decls()) {
35 if (!I->hasAttr<OMPCaptureNoInitAttr>())
36 CGF.EmitVarDecl(cast<VarDecl>(*I));
38 CodeGenFunction::AutoVarEmission Emission =
39 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40 CGF.EmitAutoVarCleanups(Emission);
47 CodeGenFunction::OMPPrivateScope InlinedShareds;
49 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50 return CGF.LambdaCaptureFields.lookup(VD) ||
51 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
56 OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S,
57 bool AsInlined = false, bool EmitPreInitStmt = true)
58 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
61 emitPreInitStmt(CGF, S);
63 if (S.hasAssociatedStmt()) {
64 auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
65 for (auto &C : CS->captures()) {
66 if (C.capturesVariable() || C.capturesVariableByCopy()) {
67 auto *VD = C.getCapturedVar();
68 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
69 isCapturedVar(CGF, VD) ||
70 (CGF.CapturedStmtInfo &&
71 InlinedShareds.isGlobalVarCaptured(VD)),
72 VD->getType().getNonReferenceType(), VK_LValue,
74 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
75 return CGF.EmitLValue(&DRE).getAddress();
79 (void)InlinedShareds.Privatize();
85 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
86 /// for captured expressions.
87 class OMPParallelScope final : public OMPLexicalScope {
88 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
89 OpenMPDirectiveKind Kind = S.getDirectiveKind();
90 return !(isOpenMPTargetExecutionDirective(Kind) ||
91 isOpenMPLoopBoundSharingDirective(Kind)) &&
92 isOpenMPParallelDirective(Kind);
96 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
97 : OMPLexicalScope(CGF, S,
99 /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
102 /// Lexical scope for OpenMP teams construct, that handles correct codegen
103 /// for captured expressions.
104 class OMPTeamsScope final : public OMPLexicalScope {
105 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
106 OpenMPDirectiveKind Kind = S.getDirectiveKind();
107 return !isOpenMPTargetExecutionDirective(Kind) &&
108 isOpenMPTeamsDirective(Kind);
112 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
113 : OMPLexicalScope(CGF, S,
115 /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
118 /// Private scope for OpenMP loop-based directives, that supports capturing
119 /// of used expression from loop statement.
120 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
121 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
122 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
123 if (auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
124 for (const auto *I : PreInits->decls())
125 CGF.EmitVarDecl(cast<VarDecl>(*I));
131 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
132 : CodeGenFunction::RunCleanupsScope(CGF) {
133 emitPreInitStmt(CGF, S);
139 llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
140 auto &C = getContext();
141 llvm::Value *Size = nullptr;
142 auto SizeInChars = C.getTypeSizeInChars(Ty);
143 if (SizeInChars.isZero()) {
144 // getTypeSizeInChars() returns 0 for a VLA.
145 while (auto *VAT = C.getAsVariableArrayType(Ty)) {
146 llvm::Value *ArraySize;
147 std::tie(ArraySize, Ty) = getVLASize(VAT);
148 Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
150 SizeInChars = C.getTypeSizeInChars(Ty);
151 if (SizeInChars.isZero())
152 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
153 Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
155 Size = CGM.getSize(SizeInChars);
159 void CodeGenFunction::GenerateOpenMPCapturedVars(
160 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
161 const RecordDecl *RD = S.getCapturedRecordDecl();
162 auto CurField = RD->field_begin();
163 auto CurCap = S.captures().begin();
164 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
165 E = S.capture_init_end();
166 I != E; ++I, ++CurField, ++CurCap) {
167 if (CurField->hasCapturedVLAType()) {
168 auto VAT = CurField->getCapturedVLAType();
169 auto *Val = VLASizeMap[VAT->getSizeExpr()];
170 CapturedVars.push_back(Val);
171 } else if (CurCap->capturesThis())
172 CapturedVars.push_back(CXXThisValue);
173 else if (CurCap->capturesVariableByCopy()) {
175 EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal();
177 // If the field is not a pointer, we need to save the actual value
178 // and load it as a void pointer.
179 if (!CurField->getType()->isAnyPointerType()) {
180 auto &Ctx = getContext();
181 auto DstAddr = CreateMemTemp(
182 Ctx.getUIntPtrType(),
183 Twine(CurCap->getCapturedVar()->getName()) + ".casted");
184 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
186 auto *SrcAddrVal = EmitScalarConversion(
187 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
188 Ctx.getPointerType(CurField->getType()), SourceLocation());
190 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
192 // Store the value using the source type pointer.
193 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
195 // Load the value using the destination type pointer.
196 CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
198 CapturedVars.push_back(CV);
200 assert(CurCap->capturesVariable() && "Expected capture by reference.");
201 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
206 static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType,
207 StringRef Name, LValue AddrLV,
208 bool isReferenceType = false) {
209 ASTContext &Ctx = CGF.getContext();
211 auto *CastedPtr = CGF.EmitScalarConversion(
212 AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
213 Ctx.getPointerType(DstType), SourceLocation());
215 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
218 // If we are dealing with references we need to return the address of the
219 // reference instead of the reference of the value.
220 if (isReferenceType) {
221 QualType RefType = Ctx.getLValueReferenceType(DstType);
222 auto *RefVal = TmpAddr.getPointer();
223 TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
224 auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
225 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true);
231 static QualType getCanonicalParamType(ASTContext &C, QualType T) {
232 if (T->isLValueReferenceType()) {
233 return C.getLValueReferenceType(
234 getCanonicalParamType(C, T.getNonReferenceType()),
235 /*SpelledAsLValue=*/false);
237 if (T->isPointerType())
238 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
239 return C.getCanonicalParamType(T);
243 /// Contains required data for proper outlined function codegen.
244 struct FunctionOptions {
245 /// Captured statement for which the function is generated.
246 const CapturedStmt *S = nullptr;
247 /// true if cast to/from UIntPtr is required for variables captured by
249 bool UIntPtrCastRequired = true;
250 /// true if only casted argumefnts must be registered as local args or VLA
252 bool RegisterCastedArgsOnly = false;
253 /// Name of the generated function.
254 StringRef FunctionName;
255 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
256 bool RegisterCastedArgsOnly,
257 StringRef FunctionName)
258 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
259 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
260 FunctionName(FunctionName) {}
264 static std::pair<llvm::Function *, bool> emitOutlinedFunctionPrologue(
265 CodeGenFunction &CGF, FunctionArgList &Args,
266 llvm::DenseMap<const Decl *, std::pair<const VarDecl *, Address>>
268 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
270 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
271 const CapturedDecl *CD = FO.S->getCapturedDecl();
272 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
273 assert(CD->hasBody() && "missing CapturedDecl body");
275 CXXThisValue = nullptr;
276 // Build the argument list.
277 CodeGenModule &CGM = CGF.CGM;
278 ASTContext &Ctx = CGM.getContext();
279 bool HasUIntPtrArgs = false;
280 Args.append(CD->param_begin(),
281 std::next(CD->param_begin(), CD->getContextParamPosition()));
282 auto I = FO.S->captures().begin();
283 for (auto *FD : RD->fields()) {
284 QualType ArgType = FD->getType();
285 IdentifierInfo *II = nullptr;
286 VarDecl *CapVar = nullptr;
288 // If this is a capture by copy and the type is not a pointer, the outlined
289 // function argument type should be uintptr and the value properly casted to
290 // uintptr. This is necessary given that the runtime library is only able to
291 // deal with pointers. We can pass in the same way the VLA type sizes to the
292 // outlined function.
293 if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
294 I->capturesVariableArrayType()) {
295 HasUIntPtrArgs = true;
296 if (FO.UIntPtrCastRequired)
297 ArgType = Ctx.getUIntPtrType();
300 if (I->capturesVariable() || I->capturesVariableByCopy()) {
301 CapVar = I->getCapturedVar();
302 II = CapVar->getIdentifier();
303 } else if (I->capturesThis())
304 II = &Ctx.Idents.get("this");
306 assert(I->capturesVariableArrayType());
307 II = &Ctx.Idents.get("vla");
309 if (ArgType->isVariablyModifiedType())
310 ArgType = getCanonicalParamType(Ctx, ArgType.getNonReferenceType());
311 Args.push_back(ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr,
312 FD->getLocation(), II, ArgType,
313 ImplicitParamDecl::Other));
317 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
320 // Create the function declaration.
321 FunctionType::ExtInfo ExtInfo;
322 const CGFunctionInfo &FuncInfo =
323 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
324 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
327 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
328 FO.FunctionName, &CGM.getModule());
329 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
331 F->addFnAttr(llvm::Attribute::NoUnwind);
333 // Generate the function.
334 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
335 CD->getBody()->getLocStart());
336 unsigned Cnt = CD->getContextParamPosition();
337 I = FO.S->captures().begin();
338 for (auto *FD : RD->fields()) {
339 // If we are capturing a pointer by copy we don't need to do anything, just
340 // use the value that we get from the arguments.
341 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
342 const VarDecl *CurVD = I->getCapturedVar();
343 Address LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
344 // If the variable is a reference we need to materialize it here.
345 if (CurVD->getType()->isReferenceType()) {
346 Address RefAddr = CGF.CreateMemTemp(
347 CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
348 CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
349 /*Volatile=*/false, CurVD->getType());
352 if (!FO.RegisterCastedArgsOnly)
353 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
359 LValueBaseInfo BaseInfo(AlignmentSource::Decl, false);
360 LValue ArgLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(Args[Cnt]),
361 Args[Cnt]->getType(), BaseInfo);
362 if (FD->hasCapturedVLAType()) {
363 if (FO.UIntPtrCastRequired) {
364 ArgLVal = CGF.MakeAddrLValue(castValueFromUintptr(CGF, FD->getType(),
365 Args[Cnt]->getName(),
367 FD->getType(), BaseInfo);
370 CGF.EmitLoadOfLValue(ArgLVal, SourceLocation()).getScalarVal();
371 auto VAT = FD->getCapturedVLAType();
372 VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}});
373 } else if (I->capturesVariable()) {
374 auto *Var = I->getCapturedVar();
375 QualType VarTy = Var->getType();
376 Address ArgAddr = ArgLVal.getAddress();
377 if (!VarTy->isReferenceType()) {
378 if (ArgLVal.getType()->isLValueReferenceType()) {
379 ArgAddr = CGF.EmitLoadOfReference(
380 ArgAddr, ArgLVal.getType()->castAs<ReferenceType>());
381 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
382 assert(ArgLVal.getType()->isPointerType());
383 ArgAddr = CGF.EmitLoadOfPointer(
384 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
387 if (!FO.RegisterCastedArgsOnly) {
390 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
392 } else if (I->capturesVariableByCopy()) {
393 assert(!FD->getType()->isAnyPointerType() &&
394 "Not expecting a captured pointer.");
395 auto *Var = I->getCapturedVar();
396 QualType VarTy = Var->getType();
400 FO.UIntPtrCastRequired
401 ? castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(),
402 ArgLVal, VarTy->isReferenceType())
403 : ArgLVal.getAddress()}});
405 // If 'this' is captured, load it into CXXThisValue.
406 assert(I->capturesThis());
407 CXXThisValue = CGF.EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation())
409 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
415 return {F, HasUIntPtrArgs};
419 CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
422 "CapturedStmtInfo should be set when generating the captured function");
423 const CapturedDecl *CD = S.getCapturedDecl();
424 // Build the argument list.
425 bool NeedWrapperFunction =
427 CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
428 FunctionArgList Args;
429 llvm::DenseMap<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
430 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
431 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
432 CapturedStmtInfo->getHelperName());
435 std::tie(F, HasUIntPtrArgs) = emitOutlinedFunctionPrologue(
436 *this, Args, LocalAddrs, VLASizes, CXXThisValue, FO);
437 for (const auto &LocalAddrPair : LocalAddrs) {
438 if (LocalAddrPair.second.first) {
439 setAddrOfLocalVar(LocalAddrPair.second.first,
440 LocalAddrPair.second.second);
443 for (const auto &VLASizePair : VLASizes)
444 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
445 PGO.assignRegionCounters(GlobalDecl(CD), F);
446 CapturedStmtInfo->EmitBody(*this, CD->getBody());
447 FinishFunction(CD->getBodyRBrace());
448 if (!NeedWrapperFunction || !HasUIntPtrArgs)
451 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
452 /*RegisterCastedArgsOnly=*/true,
453 ".nondebug_wrapper.");
454 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
455 WrapperCGF.disableDebugInfo();
459 llvm::Function *WrapperF =
460 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
461 WrapperCGF.CXXThisValue, WrapperFO).first;
462 LValueBaseInfo BaseInfo(AlignmentSource::Decl, false);
463 llvm::SmallVector<llvm::Value *, 4> CallArgs;
464 for (const auto *Arg : Args) {
465 llvm::Value *CallArg;
466 auto I = LocalAddrs.find(Arg);
467 if (I != LocalAddrs.end()) {
469 WrapperCGF.MakeAddrLValue(I->second.second, Arg->getType(), BaseInfo);
470 CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
472 auto EI = VLASizes.find(Arg);
473 if (EI != VLASizes.end())
474 CallArg = EI->second.second;
476 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
477 Arg->getType(), BaseInfo);
478 CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
481 CallArgs.emplace_back(CallArg);
483 WrapperCGF.Builder.CreateCall(F, CallArgs);
484 WrapperCGF.FinishFunction();
488 //===----------------------------------------------------------------------===//
489 // OpenMP Directive Emission
490 //===----------------------------------------------------------------------===//
491 void CodeGenFunction::EmitOMPAggregateAssign(
492 Address DestAddr, Address SrcAddr, QualType OriginalType,
493 const llvm::function_ref<void(Address, Address)> &CopyGen) {
494 // Perform element-by-element initialization.
497 // Drill down to the base element type on both arrays.
498 auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
499 auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
500 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
502 auto SrcBegin = SrcAddr.getPointer();
503 auto DestBegin = DestAddr.getPointer();
504 // Cast from pointer to array type to pointer to single element.
505 auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
506 // The basic structure here is a while-do loop.
507 auto BodyBB = createBasicBlock("omp.arraycpy.body");
508 auto DoneBB = createBasicBlock("omp.arraycpy.done");
510 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
511 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
513 // Enter the loop body, making that address the current address.
514 auto EntryBB = Builder.GetInsertBlock();
517 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
519 llvm::PHINode *SrcElementPHI =
520 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
521 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
522 Address SrcElementCurrent =
523 Address(SrcElementPHI,
524 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
526 llvm::PHINode *DestElementPHI =
527 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
528 DestElementPHI->addIncoming(DestBegin, EntryBB);
529 Address DestElementCurrent =
530 Address(DestElementPHI,
531 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
534 CopyGen(DestElementCurrent, SrcElementCurrent);
536 // Shift the address forward by one element.
537 auto DestElementNext = Builder.CreateConstGEP1_32(
538 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
539 auto SrcElementNext = Builder.CreateConstGEP1_32(
540 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
541 // Check whether we've reached the end.
543 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
544 Builder.CreateCondBr(Done, DoneBB, BodyBB);
545 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
546 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
549 EmitBlock(DoneBB, /*IsFinished=*/true);
552 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
553 Address SrcAddr, const VarDecl *DestVD,
554 const VarDecl *SrcVD, const Expr *Copy) {
555 if (OriginalType->isArrayType()) {
556 auto *BO = dyn_cast<BinaryOperator>(Copy);
557 if (BO && BO->getOpcode() == BO_Assign) {
558 // Perform simple memcpy for simple copying.
559 EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
561 // For arrays with complex element types perform element by element
563 EmitOMPAggregateAssign(
564 DestAddr, SrcAddr, OriginalType,
565 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
566 // Working with the single array element, so have to remap
567 // destination and source variables to corresponding array
569 CodeGenFunction::OMPPrivateScope Remap(*this);
570 Remap.addPrivate(DestVD, [DestElement]() -> Address {
574 SrcVD, [SrcElement]() -> Address { return SrcElement; });
575 (void)Remap.Privatize();
576 EmitIgnoredExpr(Copy);
580 // Remap pseudo source variable to private copy.
581 CodeGenFunction::OMPPrivateScope Remap(*this);
582 Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
583 Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
584 (void)Remap.Privatize();
585 // Emit copying of the whole variable.
586 EmitIgnoredExpr(Copy);
590 bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
591 OMPPrivateScope &PrivateScope) {
592 if (!HaveInsertPoint())
594 bool FirstprivateIsLastprivate = false;
595 llvm::DenseSet<const VarDecl *> Lastprivates;
596 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
597 for (const auto *D : C->varlists())
599 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
601 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
602 CGCapturedStmtInfo CapturesInfo(cast<CapturedStmt>(*D.getAssociatedStmt()));
603 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
604 auto IRef = C->varlist_begin();
605 auto InitsRef = C->inits().begin();
606 for (auto IInit : C->private_copies()) {
607 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
608 bool ThisFirstprivateIsLastprivate =
609 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
610 auto *CapFD = CapturesInfo.lookup(OrigVD);
611 auto *FD = CapturedStmtInfo->lookup(OrigVD);
612 if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
613 !FD->getType()->isReferenceType()) {
614 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
619 FirstprivateIsLastprivate =
620 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
621 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
622 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
623 auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
625 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
626 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
627 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
628 Address OriginalAddr = EmitLValue(&DRE).getAddress();
629 QualType Type = VD->getType();
630 if (Type->isArrayType()) {
631 // Emit VarDecl with copy init for arrays.
632 // Get the address of the original variable captured in current
634 IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
635 auto Emission = EmitAutoVarAlloca(*VD);
636 auto *Init = VD->getInit();
637 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
638 // Perform simple memcpy.
639 EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
642 EmitOMPAggregateAssign(
643 Emission.getAllocatedAddress(), OriginalAddr, Type,
644 [this, VDInit, Init](Address DestElement,
645 Address SrcElement) {
646 // Clean up any temporaries needed by the initialization.
647 RunCleanupsScope InitScope(*this);
648 // Emit initialization for single element.
649 setAddrOfLocalVar(VDInit, SrcElement);
650 EmitAnyExprToMem(Init, DestElement,
651 Init->getType().getQualifiers(),
652 /*IsInitializer*/ false);
653 LocalDeclMap.erase(VDInit);
656 EmitAutoVarCleanups(Emission);
657 return Emission.getAllocatedAddress();
660 IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
661 // Emit private VarDecl with copy init.
662 // Remap temp VDInit variable to the address of the original
664 // (for proper handling of captured global variables).
665 setAddrOfLocalVar(VDInit, OriginalAddr);
667 LocalDeclMap.erase(VDInit);
668 return GetAddrOfLocalVar(VD);
671 assert(IsRegistered &&
672 "firstprivate var already registered as private");
673 // Silence the warning about unused variable.
680 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
683 void CodeGenFunction::EmitOMPPrivateClause(
684 const OMPExecutableDirective &D,
685 CodeGenFunction::OMPPrivateScope &PrivateScope) {
686 if (!HaveInsertPoint())
688 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
689 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
690 auto IRef = C->varlist_begin();
691 for (auto IInit : C->private_copies()) {
692 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
693 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
694 auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
696 PrivateScope.addPrivate(OrigVD, [&]() -> Address {
697 // Emit private VarDecl with copy init.
699 return GetAddrOfLocalVar(VD);
701 assert(IsRegistered && "private var already registered as private");
702 // Silence the warning about unused variable.
710 bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
711 if (!HaveInsertPoint())
713 // threadprivate_var1 = master_threadprivate_var1;
714 // operator=(threadprivate_var2, master_threadprivate_var2);
716 // __kmpc_barrier(&loc, global_tid);
717 llvm::DenseSet<const VarDecl *> CopiedVars;
718 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
719 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
720 auto IRef = C->varlist_begin();
721 auto ISrcRef = C->source_exprs().begin();
722 auto IDestRef = C->destination_exprs().begin();
723 for (auto *AssignOp : C->assignment_ops()) {
724 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
725 QualType Type = VD->getType();
726 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
727 // Get the address of the master variable. If we are emitting code with
728 // TLS support, the address is passed from the master as field in the
729 // captured declaration.
730 Address MasterAddr = Address::invalid();
731 if (getLangOpts().OpenMPUseTLS &&
732 getContext().getTargetInfo().isTLSSupported()) {
733 assert(CapturedStmtInfo->lookup(VD) &&
734 "Copyin threadprivates should have been captured!");
735 DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
736 VK_LValue, (*IRef)->getExprLoc());
737 MasterAddr = EmitLValue(&DRE).getAddress();
738 LocalDeclMap.erase(VD);
741 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
742 : CGM.GetAddrOfGlobal(VD),
743 getContext().getDeclAlign(VD));
745 // Get the address of the threadprivate variable.
746 Address PrivateAddr = EmitLValue(*IRef).getAddress();
747 if (CopiedVars.size() == 1) {
748 // At first check if current thread is a master thread. If it is, no
749 // need to copy data.
750 CopyBegin = createBasicBlock("copyin.not.master");
751 CopyEnd = createBasicBlock("copyin.not.master.end");
752 Builder.CreateCondBr(
753 Builder.CreateICmpNE(
754 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
755 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
757 EmitBlock(CopyBegin);
759 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
760 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
761 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
769 // Exit out of copying procedure for non-master thread.
770 EmitBlock(CopyEnd, /*IsFinished=*/true);
776 bool CodeGenFunction::EmitOMPLastprivateClauseInit(
777 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
778 if (!HaveInsertPoint())
780 bool HasAtLeastOneLastprivate = false;
781 llvm::DenseSet<const VarDecl *> SIMDLCVs;
782 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
783 auto *LoopDirective = cast<OMPLoopDirective>(&D);
784 for (auto *C : LoopDirective->counters()) {
786 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
789 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
790 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
791 HasAtLeastOneLastprivate = true;
792 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()))
794 auto IRef = C->varlist_begin();
795 auto IDestRef = C->destination_exprs().begin();
796 for (auto *IInit : C->private_copies()) {
797 // Keep the address of the original variable for future update at the end
799 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
800 // Taskloops do not require additional initialization, it is done in
801 // runtime support library.
802 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
803 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
804 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
806 const_cast<VarDecl *>(OrigVD),
807 /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
809 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
810 return EmitLValue(&DRE).getAddress();
812 // Check if the variable is also a firstprivate: in this case IInit is
813 // not generated. Initialization of this variable will happen in codegen
814 // for 'firstprivate' clause.
815 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
816 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
817 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
818 // Emit private VarDecl with copy init.
820 return GetAddrOfLocalVar(VD);
822 assert(IsRegistered &&
823 "lastprivate var already registered as private");
831 return HasAtLeastOneLastprivate;
834 void CodeGenFunction::EmitOMPLastprivateClauseFinal(
835 const OMPExecutableDirective &D, bool NoFinals,
836 llvm::Value *IsLastIterCond) {
837 if (!HaveInsertPoint())
839 // Emit following code:
840 // if (<IsLastIterCond>) {
841 // orig_var1 = private_orig_var1;
843 // orig_varn = private_orig_varn;
845 llvm::BasicBlock *ThenBB = nullptr;
846 llvm::BasicBlock *DoneBB = nullptr;
847 if (IsLastIterCond) {
848 ThenBB = createBasicBlock(".omp.lastprivate.then");
849 DoneBB = createBasicBlock(".omp.lastprivate.done");
850 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
853 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
854 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
855 if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
856 auto IC = LoopDirective->counters().begin();
857 for (auto F : LoopDirective->finals()) {
859 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
861 AlreadyEmittedVars.insert(D);
863 LoopCountersAndUpdates[D] = F;
867 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
868 auto IRef = C->varlist_begin();
869 auto ISrcRef = C->source_exprs().begin();
870 auto IDestRef = C->destination_exprs().begin();
871 for (auto *AssignOp : C->assignment_ops()) {
872 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
873 QualType Type = PrivateVD->getType();
874 auto *CanonicalVD = PrivateVD->getCanonicalDecl();
875 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
876 // If lastprivate variable is a loop control variable for loop-based
877 // directive, update its value before copyin back to original
879 if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
880 EmitIgnoredExpr(FinalExpr);
881 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
882 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
883 // Get the address of the original variable.
884 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
885 // Get the address of the private variable.
886 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
887 if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
889 Address(Builder.CreateLoad(PrivateAddr),
890 getNaturalTypeAlignment(RefTy->getPointeeType()));
891 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
897 if (auto *PostUpdate = C->getPostUpdateExpr())
898 EmitIgnoredExpr(PostUpdate);
901 EmitBlock(DoneBB, /*IsFinished=*/true);
904 void CodeGenFunction::EmitOMPReductionClauseInit(
905 const OMPExecutableDirective &D,
906 CodeGenFunction::OMPPrivateScope &PrivateScope) {
907 if (!HaveInsertPoint())
909 SmallVector<const Expr *, 4> Shareds;
910 SmallVector<const Expr *, 4> Privates;
911 SmallVector<const Expr *, 4> ReductionOps;
912 SmallVector<const Expr *, 4> LHSs;
913 SmallVector<const Expr *, 4> RHSs;
914 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
915 auto IPriv = C->privates().begin();
916 auto IRed = C->reduction_ops().begin();
917 auto ILHS = C->lhs_exprs().begin();
918 auto IRHS = C->rhs_exprs().begin();
919 for (const auto *Ref : C->varlists()) {
920 Shareds.emplace_back(Ref);
921 Privates.emplace_back(*IPriv);
922 ReductionOps.emplace_back(*IRed);
923 LHSs.emplace_back(*ILHS);
924 RHSs.emplace_back(*IRHS);
925 std::advance(IPriv, 1);
926 std::advance(IRed, 1);
927 std::advance(ILHS, 1);
928 std::advance(IRHS, 1);
931 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
933 auto ILHS = LHSs.begin();
934 auto IRHS = RHSs.begin();
935 auto IPriv = Privates.begin();
936 for (const auto *IRef : Shareds) {
937 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
938 // Emit private VarDecl with reduction init.
939 RedCG.emitSharedLValue(*this, Count);
940 RedCG.emitAggregateType(*this, Count);
941 auto Emission = EmitAutoVarAlloca(*PrivateVD);
942 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
943 RedCG.getSharedLValue(Count),
944 [&Emission](CodeGenFunction &CGF) {
945 CGF.EmitAutoVarInit(Emission);
948 EmitAutoVarCleanups(Emission);
949 Address BaseAddr = RedCG.adjustPrivateAddress(
950 *this, Count, Emission.getAllocatedAddress());
951 bool IsRegistered = PrivateScope.addPrivate(
952 RedCG.getBaseDecl(Count), [BaseAddr]() -> Address { return BaseAddr; });
953 assert(IsRegistered && "private var already registered as private");
954 // Silence the warning about unused variable.
957 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
958 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
959 if (isa<OMPArraySectionExpr>(IRef)) {
960 // Store the address of the original variable associated with the LHS
961 // implicit variable.
962 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
963 return RedCG.getSharedLValue(Count).getAddress();
965 PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
966 return GetAddrOfLocalVar(PrivateVD);
968 } else if (isa<ArraySubscriptExpr>(IRef)) {
969 // Store the address of the original variable associated with the LHS
970 // implicit variable.
971 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
972 return RedCG.getSharedLValue(Count).getAddress();
974 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address {
975 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
976 ConvertTypeForMem(RHSVD->getType()),
980 QualType Type = PrivateVD->getType();
981 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
982 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
983 // Store the address of the original variable associated with the LHS
984 // implicit variable.
986 OriginalAddr = Builder.CreateElementBitCast(
987 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
989 PrivateScope.addPrivate(
990 LHSVD, [OriginalAddr]() -> Address { return OriginalAddr; });
991 PrivateScope.addPrivate(
992 RHSVD, [this, PrivateVD, RHSVD, IsArray]() -> Address {
994 ? Builder.CreateElementBitCast(
995 GetAddrOfLocalVar(PrivateVD),
996 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
997 : GetAddrOfLocalVar(PrivateVD);
1007 void CodeGenFunction::EmitOMPReductionClauseFinal(
1008 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1009 if (!HaveInsertPoint())
1011 llvm::SmallVector<const Expr *, 8> Privates;
1012 llvm::SmallVector<const Expr *, 8> LHSExprs;
1013 llvm::SmallVector<const Expr *, 8> RHSExprs;
1014 llvm::SmallVector<const Expr *, 8> ReductionOps;
1015 bool HasAtLeastOneReduction = false;
1016 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1017 HasAtLeastOneReduction = true;
1018 Privates.append(C->privates().begin(), C->privates().end());
1019 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1020 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1021 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1023 if (HasAtLeastOneReduction) {
1024 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1025 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1026 D.getDirectiveKind() == OMPD_simd;
1027 bool SimpleReduction = D.getDirectiveKind() == OMPD_simd;
1028 // Emit nowait reduction if nowait clause is present or directive is a
1029 // parallel directive (it always has implicit barrier).
1030 CGM.getOpenMPRuntime().emitReduction(
1031 *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
1032 {WithNowait, SimpleReduction, ReductionKind});
1036 static void emitPostUpdateForReductionClause(
1037 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1038 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1039 if (!CGF.HaveInsertPoint())
1041 llvm::BasicBlock *DoneBB = nullptr;
1042 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1043 if (auto *PostUpdate = C->getPostUpdateExpr()) {
1045 if (auto *Cond = CondGen(CGF)) {
1046 // If the first post-update expression is found, emit conditional
1047 // block if it was requested.
1048 auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1049 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1050 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1051 CGF.EmitBlock(ThenBB);
1054 CGF.EmitIgnoredExpr(PostUpdate);
1058 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1062 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1063 /// parallel function. This is necessary for combined constructs such as
1064 /// 'distribute parallel for'
1065 typedef llvm::function_ref<void(CodeGenFunction &,
1066 const OMPExecutableDirective &,
1067 llvm::SmallVectorImpl<llvm::Value *> &)>
1068 CodeGenBoundParametersTy;
1069 } // anonymous namespace
1071 static void emitCommonOMPParallelDirective(
1072 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1073 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1074 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1075 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1076 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1077 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1078 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1079 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1080 auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1081 /*IgnoreResultAssign*/ true);
1082 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1083 CGF, NumThreads, NumThreadsClause->getLocStart());
1085 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1086 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1087 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1088 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1090 const Expr *IfCond = nullptr;
1091 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1092 if (C->getNameModifier() == OMPD_unknown ||
1093 C->getNameModifier() == OMPD_parallel) {
1094 IfCond = C->getCondition();
1099 OMPParallelScope Scope(CGF, S);
1100 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1101 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1102 // lower and upper bounds with the pragma 'for' chunking mechanism.
1103 // The following lambda takes care of appending the lower and upper bound
1104 // parameters when necessary
1105 CodeGenBoundParameters(CGF, S, CapturedVars);
1106 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1107 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1108 CapturedVars, IfCond);
1111 static void emitEmptyBoundParameters(CodeGenFunction &,
1112 const OMPExecutableDirective &,
1113 llvm::SmallVectorImpl<llvm::Value *> &) {}
1115 void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1116 // Emit parallel region as a standalone region.
1117 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1118 OMPPrivateScope PrivateScope(CGF);
1119 bool Copyins = CGF.EmitOMPCopyinClause(S);
1120 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1122 // Emit implicit barrier to synchronize threads and avoid data races on
1123 // propagation master's thread values of threadprivate variables to local
1124 // instances of that variables of all other implicit threads.
1125 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1126 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1127 /*ForceSimpleCall=*/true);
1129 CGF.EmitOMPPrivateClause(S, PrivateScope);
1130 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1131 (void)PrivateScope.Privatize();
1132 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1133 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1135 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1136 emitEmptyBoundParameters);
1137 emitPostUpdateForReductionClause(
1138 *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1141 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1142 JumpDest LoopExit) {
1143 RunCleanupsScope BodyScope(*this);
1144 // Update counters values on current iteration.
1145 for (auto I : D.updates()) {
1148 // Update the linear variables.
1149 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1150 for (auto *U : C->updates())
1154 // On a continue in the body, jump to the end.
1155 auto Continue = getJumpDestInCurrentScope("omp.body.continue");
1156 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1158 EmitStmt(D.getBody());
1159 // The end (updates/cleanups).
1160 EmitBlock(Continue.getBlock());
1161 BreakContinueStack.pop_back();
1164 void CodeGenFunction::EmitOMPInnerLoop(
1165 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1166 const Expr *IncExpr,
1167 const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
1168 const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
1169 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1171 // Start the loop with a block that tests the condition.
1172 auto CondBlock = createBasicBlock("omp.inner.for.cond");
1173 EmitBlock(CondBlock);
1174 const SourceRange &R = S.getSourceRange();
1175 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1176 SourceLocToDebugLoc(R.getEnd()));
1178 // If there are any cleanups between here and the loop-exit scope,
1179 // create a block to stage a loop exit along.
1180 auto ExitBlock = LoopExit.getBlock();
1181 if (RequiresCleanup)
1182 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1184 auto LoopBody = createBasicBlock("omp.inner.for.body");
1187 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1188 if (ExitBlock != LoopExit.getBlock()) {
1189 EmitBlock(ExitBlock);
1190 EmitBranchThroughCleanup(LoopExit);
1193 EmitBlock(LoopBody);
1194 incrementProfileCounter(&S);
1196 // Create a block for the increment.
1197 auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1198 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1202 // Emit "IV = IV + 1" and a back-edge to the condition block.
1203 EmitBlock(Continue.getBlock());
1204 EmitIgnoredExpr(IncExpr);
1206 BreakContinueStack.pop_back();
1207 EmitBranch(CondBlock);
1209 // Emit the fall-through block.
1210 EmitBlock(LoopExit.getBlock());
1213 void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
1214 if (!HaveInsertPoint())
1216 // Emit inits for the linear variables.
1217 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1218 for (auto *Init : C->inits()) {
1219 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1220 if (auto *Ref = dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1221 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1222 auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1223 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1224 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1225 VD->getInit()->getType(), VK_LValue,
1226 VD->getInit()->getExprLoc());
1227 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1229 /*capturedByInit=*/false);
1230 EmitAutoVarCleanups(Emission);
1234 // Emit the linear steps for the linear clauses.
1235 // If a step is not constant, it is pre-calculated before the loop.
1236 if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1237 if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1238 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1239 // Emit calculation of the linear step.
1240 EmitIgnoredExpr(CS);
1245 void CodeGenFunction::EmitOMPLinearClauseFinal(
1246 const OMPLoopDirective &D,
1247 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1248 if (!HaveInsertPoint())
1250 llvm::BasicBlock *DoneBB = nullptr;
1251 // Emit the final values of the linear variables.
1252 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1253 auto IC = C->varlist_begin();
1254 for (auto *F : C->finals()) {
1256 if (auto *Cond = CondGen(*this)) {
1257 // If the first post-update expression is found, emit conditional
1258 // block if it was requested.
1259 auto *ThenBB = createBasicBlock(".omp.linear.pu");
1260 DoneBB = createBasicBlock(".omp.linear.pu.done");
1261 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1265 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1266 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1267 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1268 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1269 Address OrigAddr = EmitLValue(&DRE).getAddress();
1270 CodeGenFunction::OMPPrivateScope VarScope(*this);
1271 VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; });
1272 (void)VarScope.Privatize();
1276 if (auto *PostUpdate = C->getPostUpdateExpr())
1277 EmitIgnoredExpr(PostUpdate);
1280 EmitBlock(DoneBB, /*IsFinished=*/true);
1283 static void emitAlignedClause(CodeGenFunction &CGF,
1284 const OMPExecutableDirective &D) {
1285 if (!CGF.HaveInsertPoint())
1287 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1288 unsigned ClauseAlignment = 0;
1289 if (auto AlignmentExpr = Clause->getAlignment()) {
1291 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1292 ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1294 for (auto E : Clause->varlists()) {
1295 unsigned Alignment = ClauseAlignment;
1296 if (Alignment == 0) {
1297 // OpenMP [2.8.1, Description]
1298 // If no optional parameter is specified, implementation-defined default
1299 // alignments for SIMD instructions on the target platforms are assumed.
1302 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1303 E->getType()->getPointeeType()))
1306 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1307 "alignment is not power of 2");
1308 if (Alignment != 0) {
1309 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1310 CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1316 void CodeGenFunction::EmitOMPPrivateLoopCounters(
1317 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1318 if (!HaveInsertPoint())
1320 auto I = S.private_counters().begin();
1321 for (auto *E : S.counters()) {
1322 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1323 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1324 (void)LoopScope.addPrivate(VD, [&]() -> Address {
1325 // Emit var without initialization.
1326 if (!LocalDeclMap.count(PrivateVD)) {
1327 auto VarEmission = EmitAutoVarAlloca(*PrivateVD);
1328 EmitAutoVarCleanups(VarEmission);
1330 DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1331 /*RefersToEnclosingVariableOrCapture=*/false,
1332 (*I)->getType(), VK_LValue, (*I)->getExprLoc());
1333 return EmitLValue(&DRE).getAddress();
1335 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1336 VD->hasGlobalStorage()) {
1337 (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
1338 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1339 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1340 E->getType(), VK_LValue, E->getExprLoc());
1341 return EmitLValue(&DRE).getAddress();
1348 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1349 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1350 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1351 if (!CGF.HaveInsertPoint())
1354 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1355 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1356 (void)PreCondScope.Privatize();
1357 // Get initial values of real counters.
1358 for (auto I : S.inits()) {
1359 CGF.EmitIgnoredExpr(I);
1362 // Check that loop is executed at least one time.
1363 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1366 void CodeGenFunction::EmitOMPLinearClause(
1367 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1368 if (!HaveInsertPoint())
1370 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1371 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1372 auto *LoopDirective = cast<OMPLoopDirective>(&D);
1373 for (auto *C : LoopDirective->counters()) {
1375 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1378 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1379 auto CurPrivate = C->privates().begin();
1380 for (auto *E : C->varlists()) {
1381 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1383 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1384 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1385 bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
1386 // Emit private VarDecl with copy init.
1387 EmitVarDecl(*PrivateVD);
1388 return GetAddrOfLocalVar(PrivateVD);
1390 assert(IsRegistered && "linear var already registered as private");
1391 // Silence the warning about unused variable.
1394 EmitVarDecl(*PrivateVD);
1400 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1401 const OMPExecutableDirective &D,
1403 if (!CGF.HaveInsertPoint())
1405 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1406 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1407 /*ignoreResult=*/true);
1408 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1409 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1410 // In presence of finite 'safelen', it may be unsafe to mark all
1411 // the memory instructions parallel, because loop-carried
1412 // dependences of 'safelen' iterations are possible.
1414 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1415 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1416 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1417 /*ignoreResult=*/true);
1418 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1419 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1420 // In presence of finite 'safelen', it may be unsafe to mark all
1421 // the memory instructions parallel, because loop-carried
1422 // dependences of 'safelen' iterations are possible.
1423 CGF.LoopStack.setParallel(false);
1427 void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1429 // Walk clauses and process safelen/lastprivate.
1430 LoopStack.setParallel(!IsMonotonic);
1431 LoopStack.setVectorizeEnable(true);
1432 emitSimdlenSafelenClause(*this, D, IsMonotonic);
1435 void CodeGenFunction::EmitOMPSimdFinal(
1436 const OMPLoopDirective &D,
1437 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1438 if (!HaveInsertPoint())
1440 llvm::BasicBlock *DoneBB = nullptr;
1441 auto IC = D.counters().begin();
1442 auto IPC = D.private_counters().begin();
1443 for (auto F : D.finals()) {
1444 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1445 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1446 auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1447 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1448 OrigVD->hasGlobalStorage() || CED) {
1450 if (auto *Cond = CondGen(*this)) {
1451 // If the first post-update expression is found, emit conditional
1452 // block if it was requested.
1453 auto *ThenBB = createBasicBlock(".omp.final.then");
1454 DoneBB = createBasicBlock(".omp.final.done");
1455 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1459 Address OrigAddr = Address::invalid();
1461 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1463 DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1464 /*RefersToEnclosingVariableOrCapture=*/false,
1465 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1466 OrigAddr = EmitLValue(&DRE).getAddress();
1468 OMPPrivateScope VarScope(*this);
1469 VarScope.addPrivate(OrigVD,
1470 [OrigAddr]() -> Address { return OrigAddr; });
1471 (void)VarScope.Privatize();
1478 EmitBlock(DoneBB, /*IsFinished=*/true);
1481 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1482 const OMPLoopDirective &S,
1483 CodeGenFunction::JumpDest LoopExit) {
1484 CGF.EmitOMPLoopBody(S, LoopExit);
1485 CGF.EmitStopPoint(&S);
1488 void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1489 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1490 OMPLoopScope PreInitScope(CGF, S);
1492 // for (IV in 0..LastIteration) BODY;
1493 // <Final counter/linear vars updates>;
1497 // Emit: if (PreCond) - begin.
1498 // If the condition constant folds and can be elided, avoid emitting the
1501 llvm::BasicBlock *ContBlock = nullptr;
1502 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1506 auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1507 ContBlock = CGF.createBasicBlock("simd.if.end");
1508 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1509 CGF.getProfileCount(&S));
1510 CGF.EmitBlock(ThenBlock);
1511 CGF.incrementProfileCounter(&S);
1514 // Emit the loop iteration variable.
1515 const Expr *IVExpr = S.getIterationVariable();
1516 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1517 CGF.EmitVarDecl(*IVDecl);
1518 CGF.EmitIgnoredExpr(S.getInit());
1520 // Emit the iterations count variable.
1521 // If it is not a variable, Sema decided to calculate iterations count on
1522 // each iteration (e.g., it is foldable into a constant).
1523 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1524 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1525 // Emit calculation of the iterations count.
1526 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1529 CGF.EmitOMPSimdInit(S);
1531 emitAlignedClause(CGF, S);
1532 CGF.EmitOMPLinearClauseInit(S);
1534 OMPPrivateScope LoopScope(CGF);
1535 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1536 CGF.EmitOMPLinearClause(S, LoopScope);
1537 CGF.EmitOMPPrivateClause(S, LoopScope);
1538 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1539 bool HasLastprivateClause =
1540 CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1541 (void)LoopScope.Privatize();
1542 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1544 [&S](CodeGenFunction &CGF) {
1545 CGF.EmitOMPLoopBody(S, JumpDest());
1546 CGF.EmitStopPoint(&S);
1548 [](CodeGenFunction &) {});
1549 CGF.EmitOMPSimdFinal(
1550 S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1551 // Emit final copy of the lastprivate variables at the end of loops.
1552 if (HasLastprivateClause)
1553 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1554 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1555 emitPostUpdateForReductionClause(
1556 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1558 CGF.EmitOMPLinearClauseFinal(
1559 S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1560 // Emit: if (PreCond) - end.
1562 CGF.EmitBranch(ContBlock);
1563 CGF.EmitBlock(ContBlock, true);
1566 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1567 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1570 void CodeGenFunction::EmitOMPOuterLoop(
1571 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1572 CodeGenFunction::OMPPrivateScope &LoopScope,
1573 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1574 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1575 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1576 auto &RT = CGM.getOpenMPRuntime();
1578 const Expr *IVExpr = S.getIterationVariable();
1579 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1580 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1582 auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1584 // Start the loop with a block that tests the condition.
1585 auto CondBlock = createBasicBlock("omp.dispatch.cond");
1586 EmitBlock(CondBlock);
1587 const SourceRange &R = S.getSourceRange();
1588 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1589 SourceLocToDebugLoc(R.getEnd()));
1591 llvm::Value *BoolCondVal = nullptr;
1592 if (!DynamicOrOrdered) {
1593 // UB = min(UB, GlobalUB) or
1594 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1595 // 'distribute parallel for')
1596 EmitIgnoredExpr(LoopArgs.EUB);
1598 EmitIgnoredExpr(LoopArgs.Init);
1600 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1603 RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1604 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1607 // If there are any cleanups between here and the loop-exit scope,
1608 // create a block to stage a loop exit along.
1609 auto ExitBlock = LoopExit.getBlock();
1610 if (LoopScope.requiresCleanups())
1611 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1613 auto LoopBody = createBasicBlock("omp.dispatch.body");
1614 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1615 if (ExitBlock != LoopExit.getBlock()) {
1616 EmitBlock(ExitBlock);
1617 EmitBranchThroughCleanup(LoopExit);
1619 EmitBlock(LoopBody);
1621 // Emit "IV = LB" (in case of static schedule, we have already calculated new
1622 // LB for loop condition and emitted it above).
1623 if (DynamicOrOrdered)
1624 EmitIgnoredExpr(LoopArgs.Init);
1626 // Create a block for the increment.
1627 auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1628 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1630 // Generate !llvm.loop.parallel metadata for loads and stores for loops
1631 // with dynamic/guided scheduling and without ordered clause.
1632 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
1633 LoopStack.setParallel(!IsMonotonic);
1635 EmitOMPSimdInit(S, IsMonotonic);
1637 SourceLocation Loc = S.getLocStart();
1639 // when 'distribute' is not combined with a 'for':
1640 // while (idx <= UB) { BODY; ++idx; }
1641 // when 'distribute' is combined with a 'for'
1642 // (e.g. 'distribute parallel for')
1643 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1645 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1646 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1647 CodeGenLoop(CGF, S, LoopExit);
1649 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1650 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1653 EmitBlock(Continue.getBlock());
1654 BreakContinueStack.pop_back();
1655 if (!DynamicOrOrdered) {
1656 // Emit "LB = LB + Stride", "UB = UB + Stride".
1657 EmitIgnoredExpr(LoopArgs.NextLB);
1658 EmitIgnoredExpr(LoopArgs.NextUB);
1661 EmitBranch(CondBlock);
1663 // Emit the fall-through block.
1664 EmitBlock(LoopExit.getBlock());
1666 // Tell the runtime we are done.
1667 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1668 if (!DynamicOrOrdered)
1669 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd());
1671 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1674 void CodeGenFunction::EmitOMPForOuterLoop(
1675 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1676 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1677 const OMPLoopArguments &LoopArgs,
1678 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1679 auto &RT = CGM.getOpenMPRuntime();
1681 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1682 const bool DynamicOrOrdered =
1683 Ordered || RT.isDynamic(ScheduleKind.Schedule);
1686 !RT.isStaticNonchunked(ScheduleKind.Schedule,
1687 LoopArgs.Chunk != nullptr)) &&
1688 "static non-chunked schedule does not need outer loop");
1692 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1693 // When schedule(dynamic,chunk_size) is specified, the iterations are
1694 // distributed to threads in the team in chunks as the threads request them.
1695 // Each thread executes a chunk of iterations, then requests another chunk,
1696 // until no chunks remain to be distributed. Each chunk contains chunk_size
1697 // iterations, except for the last chunk to be distributed, which may have
1698 // fewer iterations. When no chunk_size is specified, it defaults to 1.
1700 // When schedule(guided,chunk_size) is specified, the iterations are assigned
1701 // to threads in the team in chunks as the executing threads request them.
1702 // Each thread executes a chunk of iterations, then requests another chunk,
1703 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1704 // each chunk is proportional to the number of unassigned iterations divided
1705 // by the number of threads in the team, decreasing to 1. For a chunk_size
1706 // with value k (greater than 1), the size of each chunk is determined in the
1707 // same way, with the restriction that the chunks do not contain fewer than k
1708 // iterations (except for the last chunk to be assigned, which may have fewer
1709 // than k iterations).
1711 // When schedule(auto) is specified, the decision regarding scheduling is
1712 // delegated to the compiler and/or runtime system. The programmer gives the
1713 // implementation the freedom to choose any possible mapping of iterations to
1714 // threads in the team.
1716 // When schedule(runtime) is specified, the decision regarding scheduling is
1717 // deferred until run time, and the schedule and chunk size are taken from the
1718 // run-sched-var ICV. If the ICV is set to auto, the schedule is
1719 // implementation defined
1721 // while(__kmpc_dispatch_next(&LB, &UB)) {
1723 // while (idx <= UB) { BODY; ++idx;
1724 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1728 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1729 // When schedule(static, chunk_size) is specified, iterations are divided into
1730 // chunks of size chunk_size, and the chunks are assigned to the threads in
1731 // the team in a round-robin fashion in the order of the thread number.
1733 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1734 // while (idx <= UB) { BODY; ++idx; } // inner loop
1740 const Expr *IVExpr = S.getIterationVariable();
1741 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1742 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1744 if (DynamicOrOrdered) {
1745 auto DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1746 llvm::Value *LBVal = DispatchBounds.first;
1747 llvm::Value *UBVal = DispatchBounds.second;
1748 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1750 RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1751 IVSigned, Ordered, DipatchRTInputValues);
1753 RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned,
1754 Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1755 LoopArgs.ST, LoopArgs.Chunk);
1758 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1759 const unsigned IVSize,
1760 const bool IVSigned) {
1762 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1767 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1768 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1769 OuterLoopArgs.IncExpr = S.getInc();
1770 OuterLoopArgs.Init = S.getInit();
1771 OuterLoopArgs.Cond = S.getCond();
1772 OuterLoopArgs.NextLB = S.getNextLowerBound();
1773 OuterLoopArgs.NextUB = S.getNextUpperBound();
1774 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1775 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1778 static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
1779 const unsigned IVSize, const bool IVSigned) {}
1781 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1782 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1783 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1784 const CodeGenLoopTy &CodeGenLoopContent) {
1786 auto &RT = CGM.getOpenMPRuntime();
1789 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1793 const Expr *IVExpr = S.getIterationVariable();
1794 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1795 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1797 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1798 IVSigned, /* Ordered = */ false, LoopArgs.IL,
1799 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1802 // for combined 'distribute' and 'for' the increment expression of distribute
1803 // is store in DistInc. For 'distribute' alone, it is in Inc.
1805 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
1806 IncExpr = S.getDistInc();
1808 IncExpr = S.getInc();
1810 // this routine is shared by 'omp distribute parallel for' and
1811 // 'omp distribute': select the right EUB expression depending on the
1813 OMPLoopArguments OuterLoopArgs;
1814 OuterLoopArgs.LB = LoopArgs.LB;
1815 OuterLoopArgs.UB = LoopArgs.UB;
1816 OuterLoopArgs.ST = LoopArgs.ST;
1817 OuterLoopArgs.IL = LoopArgs.IL;
1818 OuterLoopArgs.Chunk = LoopArgs.Chunk;
1819 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1820 ? S.getCombinedEnsureUpperBound()
1821 : S.getEnsureUpperBound();
1822 OuterLoopArgs.IncExpr = IncExpr;
1823 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1824 ? S.getCombinedInit()
1826 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1827 ? S.getCombinedCond()
1829 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1830 ? S.getCombinedNextLowerBound()
1831 : S.getNextLowerBound();
1832 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1833 ? S.getCombinedNextUpperBound()
1834 : S.getNextUpperBound();
1836 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
1837 LoopScope, OuterLoopArgs, CodeGenLoopContent,
1841 /// Emit a helper variable and return corresponding lvalue.
1842 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1843 const DeclRefExpr *Helper) {
1844 auto VDecl = cast<VarDecl>(Helper->getDecl());
1845 CGF.EmitVarDecl(*VDecl);
1846 return CGF.EmitLValue(Helper);
1849 static std::pair<LValue, LValue>
1850 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
1851 const OMPExecutableDirective &S) {
1852 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
1854 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
1856 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
1858 // When composing 'distribute' with 'for' (e.g. as in 'distribute
1859 // parallel for') we need to use the 'distribute'
1860 // chunk lower and upper bounds rather than the whole loop iteration
1861 // space. These are parameters to the outlined function for 'parallel'
1862 // and we copy the bounds of the previous schedule into the
1863 // the current ones.
1864 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
1865 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
1866 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(PrevLB, SourceLocation());
1867 PrevLBVal = CGF.EmitScalarConversion(
1868 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
1869 LS.getIterationVariable()->getType(), SourceLocation());
1870 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(PrevUB, SourceLocation());
1871 PrevUBVal = CGF.EmitScalarConversion(
1872 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
1873 LS.getIterationVariable()->getType(), SourceLocation());
1875 CGF.EmitStoreOfScalar(PrevLBVal, LB);
1876 CGF.EmitStoreOfScalar(PrevUBVal, UB);
1881 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
1882 /// we need to use the LB and UB expressions generated by the worksharing
1883 /// code generation support, whereas in non combined situations we would
1884 /// just emit 0 and the LastIteration expression
1885 /// This function is necessary due to the difference of the LB and UB
1886 /// types for the RT emission routines for 'for_static_init' and
1887 /// 'for_dispatch_init'
1888 static std::pair<llvm::Value *, llvm::Value *>
1889 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
1890 const OMPExecutableDirective &S,
1891 Address LB, Address UB) {
1892 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
1893 const Expr *IVExpr = LS.getIterationVariable();
1894 // when implementing a dynamic schedule for a 'for' combined with a
1895 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
1896 // is not normalized as each team only executes its own assigned
1898 QualType IteratorTy = IVExpr->getType();
1899 llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy,
1901 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy,
1903 return {LBVal, UBVal};
1906 static void emitDistributeParallelForDistributeInnerBoundParams(
1907 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1908 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
1909 const auto &Dir = cast<OMPLoopDirective>(S);
1911 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
1912 auto LBCast = CGF.Builder.CreateIntCast(
1913 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
1914 CapturedVars.push_back(LBCast);
1916 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
1918 auto UBCast = CGF.Builder.CreateIntCast(
1919 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
1920 CapturedVars.push_back(UBCast);
1924 emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
1925 const OMPLoopDirective &S,
1926 CodeGenFunction::JumpDest LoopExit) {
1927 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
1928 PrePostActionTy &) {
1929 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
1930 emitDistributeParallelForInnerBounds,
1931 emitDistributeParallelForDispatchBounds);
1934 emitCommonOMPParallelDirective(
1935 CGF, S, OMPD_for, CGInlinedWorksharingLoop,
1936 emitDistributeParallelForDistributeInnerBoundParams);
1939 void CodeGenFunction::EmitOMPDistributeParallelForDirective(
1940 const OMPDistributeParallelForDirective &S) {
1941 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1942 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
1945 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1946 OMPCancelStackRAII CancelRegion(*this, OMPD_distribute_parallel_for,
1947 /*HasCancel=*/false);
1948 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen,
1949 /*HasCancel=*/false);
1952 void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
1953 const OMPDistributeParallelForSimdDirective &S) {
1954 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1955 CGM.getOpenMPRuntime().emitInlinedDirective(
1956 *this, OMPD_distribute_parallel_for_simd,
1957 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1958 OMPLoopScope PreInitScope(CGF, S);
1960 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1964 void CodeGenFunction::EmitOMPDistributeSimdDirective(
1965 const OMPDistributeSimdDirective &S) {
1966 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1967 CGM.getOpenMPRuntime().emitInlinedDirective(
1968 *this, OMPD_distribute_simd,
1969 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1970 OMPLoopScope PreInitScope(CGF, S);
1972 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1976 void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
1977 const OMPTargetParallelForSimdDirective &S) {
1978 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1979 CGM.getOpenMPRuntime().emitInlinedDirective(
1980 *this, OMPD_target_parallel_for_simd,
1981 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1982 OMPLoopScope PreInitScope(CGF, S);
1984 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1988 void CodeGenFunction::EmitOMPTargetSimdDirective(
1989 const OMPTargetSimdDirective &S) {
1990 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1991 CGM.getOpenMPRuntime().emitInlinedDirective(
1992 *this, OMPD_target_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1993 OMPLoopScope PreInitScope(CGF, S);
1995 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1999 void CodeGenFunction::EmitOMPTeamsDistributeDirective(
2000 const OMPTeamsDistributeDirective &S) {
2001 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2002 CGM.getOpenMPRuntime().emitInlinedDirective(
2003 *this, OMPD_teams_distribute,
2004 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2005 OMPLoopScope PreInitScope(CGF, S);
2007 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2011 void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
2012 const OMPTeamsDistributeSimdDirective &S) {
2013 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2014 CGM.getOpenMPRuntime().emitInlinedDirective(
2015 *this, OMPD_teams_distribute_simd,
2016 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2017 OMPLoopScope PreInitScope(CGF, S);
2019 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2023 void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
2024 const OMPTeamsDistributeParallelForSimdDirective &S) {
2025 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2026 CGM.getOpenMPRuntime().emitInlinedDirective(
2027 *this, OMPD_teams_distribute_parallel_for_simd,
2028 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2029 OMPLoopScope PreInitScope(CGF, S);
2031 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2035 void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
2036 const OMPTeamsDistributeParallelForDirective &S) {
2037 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2038 CGM.getOpenMPRuntime().emitInlinedDirective(
2039 *this, OMPD_teams_distribute_parallel_for,
2040 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2041 OMPLoopScope PreInitScope(CGF, S);
2043 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2047 void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
2048 const OMPTargetTeamsDistributeDirective &S) {
2049 CGM.getOpenMPRuntime().emitInlinedDirective(
2050 *this, OMPD_target_teams_distribute,
2051 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2053 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2057 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
2058 const OMPTargetTeamsDistributeParallelForDirective &S) {
2059 CGM.getOpenMPRuntime().emitInlinedDirective(
2060 *this, OMPD_target_teams_distribute_parallel_for,
2061 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2063 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2067 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
2068 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
2069 CGM.getOpenMPRuntime().emitInlinedDirective(
2070 *this, OMPD_target_teams_distribute_parallel_for_simd,
2071 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2073 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2077 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
2078 const OMPTargetTeamsDistributeSimdDirective &S) {
2079 CGM.getOpenMPRuntime().emitInlinedDirective(
2080 *this, OMPD_target_teams_distribute_simd,
2081 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2083 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2088 struct ScheduleKindModifiersTy {
2089 OpenMPScheduleClauseKind Kind;
2090 OpenMPScheduleClauseModifier M1;
2091 OpenMPScheduleClauseModifier M2;
2092 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2093 OpenMPScheduleClauseModifier M1,
2094 OpenMPScheduleClauseModifier M2)
2095 : Kind(Kind), M1(M1), M2(M2) {}
2099 bool CodeGenFunction::EmitOMPWorksharingLoop(
2100 const OMPLoopDirective &S, Expr *EUB,
2101 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2102 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2103 // Emit the loop iteration variable.
2104 auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2105 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2106 EmitVarDecl(*IVDecl);
2108 // Emit the iterations count variable.
2109 // If it is not a variable, Sema decided to calculate iterations count on each
2110 // iteration (e.g., it is foldable into a constant).
2111 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2112 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2113 // Emit calculation of the iterations count.
2114 EmitIgnoredExpr(S.getCalcLastIteration());
2117 auto &RT = CGM.getOpenMPRuntime();
2119 bool HasLastprivateClause;
2120 // Check pre-condition.
2122 OMPLoopScope PreInitScope(*this, S);
2123 // Skip the entire loop if we don't meet the precondition.
2124 // If the condition constant folds and can be elided, avoid emitting the
2127 llvm::BasicBlock *ContBlock = nullptr;
2128 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2132 auto *ThenBlock = createBasicBlock("omp.precond.then");
2133 ContBlock = createBasicBlock("omp.precond.end");
2134 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2135 getProfileCount(&S));
2136 EmitBlock(ThenBlock);
2137 incrementProfileCounter(&S);
2140 bool Ordered = false;
2141 if (auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2142 if (OrderedClause->getNumForLoops())
2143 RT.emitDoacrossInit(*this, S);
2148 llvm::DenseSet<const Expr *> EmittedFinals;
2149 emitAlignedClause(*this, S);
2150 EmitOMPLinearClauseInit(S);
2151 // Emit helper vars inits.
2153 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2154 LValue LB = Bounds.first;
2155 LValue UB = Bounds.second;
2157 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2159 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2161 // Emit 'then' code.
2163 OMPPrivateScope LoopScope(*this);
2164 if (EmitOMPFirstprivateClause(S, LoopScope)) {
2165 // Emit implicit barrier to synchronize threads and avoid data races on
2166 // initialization of firstprivate variables and post-update of
2167 // lastprivate variables.
2168 CGM.getOpenMPRuntime().emitBarrierCall(
2169 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2170 /*ForceSimpleCall=*/true);
2172 EmitOMPPrivateClause(S, LoopScope);
2173 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2174 EmitOMPReductionClauseInit(S, LoopScope);
2175 EmitOMPPrivateLoopCounters(S, LoopScope);
2176 EmitOMPLinearClause(S, LoopScope);
2177 (void)LoopScope.Privatize();
2179 // Detect the loop schedule kind and chunk.
2180 llvm::Value *Chunk = nullptr;
2181 OpenMPScheduleTy ScheduleKind;
2182 if (auto *C = S.getSingleClause<OMPScheduleClause>()) {
2183 ScheduleKind.Schedule = C->getScheduleKind();
2184 ScheduleKind.M1 = C->getFirstScheduleModifier();
2185 ScheduleKind.M2 = C->getSecondScheduleModifier();
2186 if (const auto *Ch = C->getChunkSize()) {
2187 Chunk = EmitScalarExpr(Ch);
2188 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2189 S.getIterationVariable()->getType(),
2193 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2194 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2195 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2196 // If the static schedule kind is specified or if the ordered clause is
2197 // specified, and if no monotonic modifier is specified, the effect will
2198 // be as if the monotonic modifier was specified.
2199 if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2200 /* Chunked */ Chunk != nullptr) &&
2202 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2203 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2204 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2205 // When no chunk_size is specified, the iteration space is divided into
2206 // chunks that are approximately equal in size, and at most one chunk is
2207 // distributed to each thread. Note that the size of the chunks is
2208 // unspecified in this case.
2209 RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind,
2210 IVSize, IVSigned, Ordered,
2211 IL.getAddress(), LB.getAddress(),
2212 UB.getAddress(), ST.getAddress());
2214 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2215 // UB = min(UB, GlobalUB);
2216 EmitIgnoredExpr(S.getEnsureUpperBound());
2218 EmitIgnoredExpr(S.getInit());
2219 // while (idx <= UB) { BODY; ++idx; }
2220 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2222 [&S, LoopExit](CodeGenFunction &CGF) {
2223 CGF.EmitOMPLoopBody(S, LoopExit);
2224 CGF.EmitStopPoint(&S);
2226 [](CodeGenFunction &) {});
2227 EmitBlock(LoopExit.getBlock());
2228 // Tell the runtime we are done.
2229 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2230 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd());
2232 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2234 const bool IsMonotonic =
2235 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2236 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2237 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2238 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2239 // Emit the outer loop, which requests its work chunk [LB..UB] from
2240 // runtime and runs the inner loop to process it.
2241 const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2242 ST.getAddress(), IL.getAddress(),
2244 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2245 LoopArguments, CGDispatchBounds);
2247 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2249 [&](CodeGenFunction &CGF) -> llvm::Value * {
2250 return CGF.Builder.CreateIsNotNull(
2251 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2254 EmitOMPReductionClauseFinal(
2255 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2256 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2257 : /*Parallel only*/ OMPD_parallel);
2258 // Emit post-update of the reduction variables if IsLastIter != 0.
2259 emitPostUpdateForReductionClause(
2260 *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2261 return CGF.Builder.CreateIsNotNull(
2262 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2264 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2265 if (HasLastprivateClause)
2266 EmitOMPLastprivateClauseFinal(
2267 S, isOpenMPSimdDirective(S.getDirectiveKind()),
2268 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
2270 EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2271 return CGF.Builder.CreateIsNotNull(
2272 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2274 // We're now done with the loop, so jump to the continuation block.
2276 EmitBranch(ContBlock);
2277 EmitBlock(ContBlock, true);
2280 return HasLastprivateClause;
2283 /// The following two functions generate expressions for the loop lower
2284 /// and upper bounds in case of static and dynamic (dispatch) schedule
2285 /// of the associated 'for' or 'distribute' loop.
2286 static std::pair<LValue, LValue>
2287 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2288 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2290 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2292 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2296 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2297 /// consider the lower and upper bound expressions generated by the
2298 /// worksharing loop support, but we use 0 and the iteration space size as
2300 static std::pair<llvm::Value *, llvm::Value *>
2301 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2302 Address LB, Address UB) {
2303 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2304 const Expr *IVExpr = LS.getIterationVariable();
2305 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2306 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2307 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2308 return {LBVal, UBVal};
2311 void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
2312 bool HasLastprivates = false;
2313 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2314 PrePostActionTy &) {
2315 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2316 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2318 emitDispatchForLoopBounds);
2321 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2322 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2326 // Emit an implicit barrier at the end.
2327 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2328 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2332 void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
2333 bool HasLastprivates = false;
2334 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2335 PrePostActionTy &) {
2336 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2338 emitDispatchForLoopBounds);
2341 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2342 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2345 // Emit an implicit barrier at the end.
2346 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2347 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2351 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2353 llvm::Value *Init = nullptr) {
2354 auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2356 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2360 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2361 auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
2362 auto *CS = dyn_cast<CompoundStmt>(Stmt);
2363 bool HasLastprivates = false;
2364 auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF,
2365 PrePostActionTy &) {
2366 auto &C = CGF.CGM.getContext();
2367 auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2368 // Emit helper vars inits.
2369 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2370 CGF.Builder.getInt32(0));
2371 auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2372 : CGF.Builder.getInt32(0);
2374 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2375 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2376 CGF.Builder.getInt32(1));
2377 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2378 CGF.Builder.getInt32(0));
2380 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2381 OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2382 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2383 OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2384 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2385 // Generate condition for loop.
2386 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2387 OK_Ordinary, S.getLocStart(), FPOptions());
2388 // Increment for loop counter.
2389 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2391 auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
2392 // Iterate through all sections and emit a switch construct:
2395 // <SectionStmt[0]>;
2398 // case <NumSection> - 1:
2399 // <SectionStmt[<NumSection> - 1]>;
2402 // .omp.sections.exit:
2403 auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2404 auto *SwitchStmt = CGF.Builder.CreateSwitch(
2405 CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2406 CS == nullptr ? 1 : CS->size());
2408 unsigned CaseNumber = 0;
2409 for (auto *SubStmt : CS->children()) {
2410 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2411 CGF.EmitBlock(CaseBB);
2412 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2413 CGF.EmitStmt(SubStmt);
2414 CGF.EmitBranch(ExitBB);
2418 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2419 CGF.EmitBlock(CaseBB);
2420 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2422 CGF.EmitBranch(ExitBB);
2424 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2427 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2428 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2429 // Emit implicit barrier to synchronize threads and avoid data races on
2430 // initialization of firstprivate variables and post-update of lastprivate
2432 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2433 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2434 /*ForceSimpleCall=*/true);
2436 CGF.EmitOMPPrivateClause(S, LoopScope);
2437 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2438 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2439 (void)LoopScope.Privatize();
2441 // Emit static non-chunked loop.
2442 OpenMPScheduleTy ScheduleKind;
2443 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2444 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2445 CGF, S.getLocStart(), ScheduleKind, /*IVSize=*/32,
2446 /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(),
2447 UB.getAddress(), ST.getAddress());
2448 // UB = min(UB, GlobalUB);
2449 auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2450 auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2451 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2452 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2454 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2455 // while (idx <= UB) { BODY; ++idx; }
2456 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2457 [](CodeGenFunction &) {});
2458 // Tell the runtime we are done.
2459 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2460 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd());
2462 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2463 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2464 // Emit post-update of the reduction variables if IsLastIter != 0.
2465 emitPostUpdateForReductionClause(
2466 CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2467 return CGF.Builder.CreateIsNotNull(
2468 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2471 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2472 if (HasLastprivates)
2473 CGF.EmitOMPLastprivateClauseFinal(
2474 S, /*NoFinals=*/false,
2475 CGF.Builder.CreateIsNotNull(
2476 CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2479 bool HasCancel = false;
2480 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2481 HasCancel = OSD->hasCancel();
2482 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2483 HasCancel = OPSD->hasCancel();
2484 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2485 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2487 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2488 // clause. Otherwise the barrier will be generated by the codegen for the
2490 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2491 // Emit implicit barrier to synchronize threads and avoid data races on
2492 // initialization of firstprivate variables.
2493 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2498 void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
2500 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2503 // Emit an implicit barrier at the end.
2504 if (!S.getSingleClause<OMPNowaitClause>()) {
2505 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2510 void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
2511 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2512 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2514 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2515 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2519 void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
2520 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2521 llvm::SmallVector<const Expr *, 8> DestExprs;
2522 llvm::SmallVector<const Expr *, 8> SrcExprs;
2523 llvm::SmallVector<const Expr *, 8> AssignmentOps;
2524 // Check if there are any 'copyprivate' clauses associated with this
2525 // 'single' construct.
2526 // Build a list of copyprivate variables along with helper expressions
2527 // (<source>, <destination>, <destination>=<source> expressions)
2528 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2529 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2530 DestExprs.append(C->destination_exprs().begin(),
2531 C->destination_exprs().end());
2532 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2533 AssignmentOps.append(C->assignment_ops().begin(),
2534 C->assignment_ops().end());
2536 // Emit code for 'single' region along with 'copyprivate' clauses
2537 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2539 OMPPrivateScope SingleScope(CGF);
2540 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2541 CGF.EmitOMPPrivateClause(S, SingleScope);
2542 (void)SingleScope.Privatize();
2543 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2546 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2547 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2548 CopyprivateVars, DestExprs,
2549 SrcExprs, AssignmentOps);
2551 // Emit an implicit barrier at the end (to avoid data race on firstprivate
2552 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2553 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2554 CGM.getOpenMPRuntime().emitBarrierCall(
2555 *this, S.getLocStart(),
2556 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2560 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
2561 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2563 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2565 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2566 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
2569 void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
2570 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2572 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2574 Expr *Hint = nullptr;
2575 if (auto *HintClause = S.getSingleClause<OMPHintClause>())
2576 Hint = HintClause->getHint();
2577 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2578 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2579 S.getDirectiveName().getAsString(),
2580 CodeGen, S.getLocStart(), Hint);
2583 void CodeGenFunction::EmitOMPParallelForDirective(
2584 const OMPParallelForDirective &S) {
2585 // Emit directive as a combined directive that consists of two implicit
2586 // directives: 'parallel' with 'for' directive.
2587 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2588 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2589 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2590 emitDispatchForLoopBounds);
2592 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2593 emitEmptyBoundParameters);
2596 void CodeGenFunction::EmitOMPParallelForSimdDirective(
2597 const OMPParallelForSimdDirective &S) {
2598 // Emit directive as a combined directive that consists of two implicit
2599 // directives: 'parallel' with 'for' directive.
2600 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2601 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2602 emitDispatchForLoopBounds);
2604 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2605 emitEmptyBoundParameters);
2608 void CodeGenFunction::EmitOMPParallelSectionsDirective(
2609 const OMPParallelSectionsDirective &S) {
2610 // Emit directive as a combined directive that consists of two implicit
2611 // directives: 'parallel' with 'sections' directive.
2612 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2613 CGF.EmitSections(S);
2615 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2616 emitEmptyBoundParameters);
2619 void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
2620 const RegionCodeGenTy &BodyGen,
2621 const TaskGenTy &TaskGen,
2622 OMPTaskDataTy &Data) {
2623 // Emit outlined function for task construct.
2624 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2625 auto *I = CS->getCapturedDecl()->param_begin();
2626 auto *PartId = std::next(I);
2627 auto *TaskT = std::next(I, 4);
2628 // Check if the task is final
2629 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2630 // If the condition constant folds and can be elided, try to avoid emitting
2631 // the condition and the dead arm of the if/else.
2632 auto *Cond = Clause->getCondition();
2634 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2635 Data.Final.setInt(CondConstant);
2637 Data.Final.setPointer(EvaluateExprAsBool(Cond));
2639 // By default the task is not final.
2640 Data.Final.setInt(/*IntVal=*/false);
2642 // Check if the task has 'priority' clause.
2643 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2644 auto *Prio = Clause->getPriority();
2645 Data.Priority.setInt(/*IntVal=*/true);
2646 Data.Priority.setPointer(EmitScalarConversion(
2647 EmitScalarExpr(Prio), Prio->getType(),
2648 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2649 Prio->getExprLoc()));
2651 // The first function argument for tasks is a thread id, the second one is a
2652 // part id (0 for tied tasks, >=0 for untied task).
2653 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2654 // Get list of private variables.
2655 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2656 auto IRef = C->varlist_begin();
2657 for (auto *IInit : C->private_copies()) {
2658 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2659 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2660 Data.PrivateVars.push_back(*IRef);
2661 Data.PrivateCopies.push_back(IInit);
2666 EmittedAsPrivate.clear();
2667 // Get list of firstprivate variables.
2668 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2669 auto IRef = C->varlist_begin();
2670 auto IElemInitRef = C->inits().begin();
2671 for (auto *IInit : C->private_copies()) {
2672 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2673 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2674 Data.FirstprivateVars.push_back(*IRef);
2675 Data.FirstprivateCopies.push_back(IInit);
2676 Data.FirstprivateInits.push_back(*IElemInitRef);
2682 // Get list of lastprivate variables (for taskloops).
2683 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2684 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2685 auto IRef = C->varlist_begin();
2686 auto ID = C->destination_exprs().begin();
2687 for (auto *IInit : C->private_copies()) {
2688 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2689 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2690 Data.LastprivateVars.push_back(*IRef);
2691 Data.LastprivateCopies.push_back(IInit);
2693 LastprivateDstsOrigs.insert(
2694 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2695 cast<DeclRefExpr>(*IRef)});
2700 SmallVector<const Expr *, 4> LHSs;
2701 SmallVector<const Expr *, 4> RHSs;
2702 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2703 auto IPriv = C->privates().begin();
2704 auto IRed = C->reduction_ops().begin();
2705 auto ILHS = C->lhs_exprs().begin();
2706 auto IRHS = C->rhs_exprs().begin();
2707 for (const auto *Ref : C->varlists()) {
2708 Data.ReductionVars.emplace_back(Ref);
2709 Data.ReductionCopies.emplace_back(*IPriv);
2710 Data.ReductionOps.emplace_back(*IRed);
2711 LHSs.emplace_back(*ILHS);
2712 RHSs.emplace_back(*IRHS);
2713 std::advance(IPriv, 1);
2714 std::advance(IRed, 1);
2715 std::advance(ILHS, 1);
2716 std::advance(IRHS, 1);
2719 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2720 *this, S.getLocStart(), LHSs, RHSs, Data);
2721 // Build list of dependences.
2722 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2723 for (auto *IRef : C->varlists())
2724 Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
2725 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs](
2726 CodeGenFunction &CGF, PrePostActionTy &Action) {
2727 // Set proper addresses for generated private copies.
2728 OMPPrivateScope Scope(CGF);
2729 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2730 !Data.LastprivateVars.empty()) {
2731 auto *CopyFn = CGF.Builder.CreateLoad(
2732 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2733 auto *PrivatesPtr = CGF.Builder.CreateLoad(
2734 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2736 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
2737 llvm::SmallVector<llvm::Value *, 16> CallArgs;
2738 CallArgs.push_back(PrivatesPtr);
2739 for (auto *E : Data.PrivateVars) {
2740 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2741 Address PrivatePtr = CGF.CreateMemTemp(
2742 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2743 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2744 CallArgs.push_back(PrivatePtr.getPointer());
2746 for (auto *E : Data.FirstprivateVars) {
2747 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2748 Address PrivatePtr =
2749 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2750 ".firstpriv.ptr.addr");
2751 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2752 CallArgs.push_back(PrivatePtr.getPointer());
2754 for (auto *E : Data.LastprivateVars) {
2755 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2756 Address PrivatePtr =
2757 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2758 ".lastpriv.ptr.addr");
2759 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2760 CallArgs.push_back(PrivatePtr.getPointer());
2762 CGF.EmitRuntimeCall(CopyFn, CallArgs);
2763 for (auto &&Pair : LastprivateDstsOrigs) {
2764 auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2766 const_cast<VarDecl *>(OrigVD),
2767 /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2769 Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2770 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2771 return CGF.EmitLValue(&DRE).getAddress();
2774 for (auto &&Pair : PrivatePtrs) {
2775 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2776 CGF.getContext().getDeclAlign(Pair.first));
2777 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2780 if (Data.Reductions) {
2781 OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true);
2782 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
2784 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2785 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2786 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2787 RedCG.emitSharedLValue(CGF, Cnt);
2788 RedCG.emitAggregateType(CGF, Cnt);
2789 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2790 CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2792 Address(CGF.EmitScalarConversion(
2793 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2794 CGF.getContext().getPointerType(
2795 Data.ReductionCopies[Cnt]->getType()),
2797 Replacement.getAlignment());
2798 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2799 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2800 [Replacement]() { return Replacement; });
2801 // FIXME: This must removed once the runtime library is fixed.
2802 // Emit required threadprivate variables for
2803 // initilizer/combiner/finalizer.
2804 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2808 (void)Scope.Privatize();
2813 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2814 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2815 Data.NumberOfParts);
2816 OMPLexicalScope Scope(*this, S);
2817 TaskGen(*this, OutlinedFn, Data);
2820 void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
2821 // Emit outlined function for task construct.
2822 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2823 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
2824 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
2825 const Expr *IfCond = nullptr;
2826 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2827 if (C->getNameModifier() == OMPD_unknown ||
2828 C->getNameModifier() == OMPD_task) {
2829 IfCond = C->getCondition();
2835 // Check if we should emit tied or untied task.
2836 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
2837 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
2838 CGF.EmitStmt(CS->getCapturedStmt());
2840 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
2841 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
2842 const OMPTaskDataTy &Data) {
2843 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
2844 SharedsTy, CapturedStruct, IfCond,
2847 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
2850 void CodeGenFunction::EmitOMPTaskyieldDirective(
2851 const OMPTaskyieldDirective &S) {
2852 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
2855 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
2856 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
2859 void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
2860 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
2863 void CodeGenFunction::EmitOMPTaskgroupDirective(
2864 const OMPTaskgroupDirective &S) {
2865 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2867 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2869 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2870 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
2873 void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
2874 CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
2875 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
2876 return llvm::makeArrayRef(FlushClause->varlist_begin(),
2877 FlushClause->varlist_end());
2880 }(), S.getLocStart());
2883 void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
2884 const CodeGenLoopTy &CodeGenLoop,
2886 // Emit the loop iteration variable.
2887 auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2888 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2889 EmitVarDecl(*IVDecl);
2891 // Emit the iterations count variable.
2892 // If it is not a variable, Sema decided to calculate iterations count on each
2893 // iteration (e.g., it is foldable into a constant).
2894 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2895 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2896 // Emit calculation of the iterations count.
2897 EmitIgnoredExpr(S.getCalcLastIteration());
2900 auto &RT = CGM.getOpenMPRuntime();
2902 bool HasLastprivateClause = false;
2903 // Check pre-condition.
2905 OMPLoopScope PreInitScope(*this, S);
2906 // Skip the entire loop if we don't meet the precondition.
2907 // If the condition constant folds and can be elided, avoid emitting the
2910 llvm::BasicBlock *ContBlock = nullptr;
2911 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2915 auto *ThenBlock = createBasicBlock("omp.precond.then");
2916 ContBlock = createBasicBlock("omp.precond.end");
2917 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2918 getProfileCount(&S));
2919 EmitBlock(ThenBlock);
2920 incrementProfileCounter(&S);
2923 // Emit 'then' code.
2925 // Emit helper vars inits.
2927 LValue LB = EmitOMPHelperVar(
2928 *this, cast<DeclRefExpr>(
2929 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2930 ? S.getCombinedLowerBoundVariable()
2931 : S.getLowerBoundVariable())));
2932 LValue UB = EmitOMPHelperVar(
2933 *this, cast<DeclRefExpr>(
2934 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2935 ? S.getCombinedUpperBoundVariable()
2936 : S.getUpperBoundVariable())));
2938 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2940 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2942 OMPPrivateScope LoopScope(*this);
2943 if (EmitOMPFirstprivateClause(S, LoopScope)) {
2944 // Emit implicit barrier to synchronize threads and avoid data races on
2945 // initialization of firstprivate variables and post-update of
2946 // lastprivate variables.
2947 CGM.getOpenMPRuntime().emitBarrierCall(
2948 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2949 /*ForceSimpleCall=*/true);
2951 EmitOMPPrivateClause(S, LoopScope);
2952 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2953 EmitOMPPrivateLoopCounters(S, LoopScope);
2954 (void)LoopScope.Privatize();
2956 // Detect the distribute schedule kind and chunk.
2957 llvm::Value *Chunk = nullptr;
2958 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
2959 if (auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
2960 ScheduleKind = C->getDistScheduleKind();
2961 if (const auto *Ch = C->getChunkSize()) {
2962 Chunk = EmitScalarExpr(Ch);
2963 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2964 S.getIterationVariable()->getType(),
2968 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2969 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2971 // OpenMP [2.10.8, distribute Construct, Description]
2972 // If dist_schedule is specified, kind must be static. If specified,
2973 // iterations are divided into chunks of size chunk_size, chunks are
2974 // assigned to the teams of the league in a round-robin fashion in the
2975 // order of the team number. When no chunk_size is specified, the
2976 // iteration space is divided into chunks that are approximately equal
2977 // in size, and at most one chunk is distributed to each team of the
2978 // league. The size of the chunks is unspecified in this case.
2979 if (RT.isStaticNonchunked(ScheduleKind,
2980 /* Chunked */ Chunk != nullptr)) {
2981 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
2982 IVSize, IVSigned, /* Ordered = */ false,
2983 IL.getAddress(), LB.getAddress(),
2984 UB.getAddress(), ST.getAddress());
2986 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2987 // UB = min(UB, GlobalUB);
2988 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2989 ? S.getCombinedEnsureUpperBound()
2990 : S.getEnsureUpperBound());
2992 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2993 ? S.getCombinedInit()
2996 Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2997 ? S.getCombinedCond()
3000 // for distribute alone, codegen
3001 // while (idx <= UB) { BODY; ++idx; }
3002 // when combined with 'for' (e.g. as in 'distribute parallel for')
3003 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3004 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3005 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3006 CodeGenLoop(CGF, S, LoopExit);
3008 [](CodeGenFunction &) {});
3009 EmitBlock(LoopExit.getBlock());
3010 // Tell the runtime we are done.
3011 RT.emitForStaticFinish(*this, S.getLocStart());
3013 // Emit the outer loop, which requests its work chunk [LB..UB] from
3014 // runtime and runs the inner loop to process it.
3015 const OMPLoopArguments LoopArguments = {
3016 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3018 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3022 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3023 if (HasLastprivateClause)
3024 EmitOMPLastprivateClauseFinal(
3025 S, /*NoFinals=*/false,
3026 Builder.CreateIsNotNull(
3027 EmitLoadOfScalar(IL, S.getLocStart())));
3030 // We're now done with the loop, so jump to the continuation block.
3032 EmitBranch(ContBlock);
3033 EmitBlock(ContBlock, true);
3038 void CodeGenFunction::EmitOMPDistributeDirective(
3039 const OMPDistributeDirective &S) {
3040 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3042 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3044 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3045 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen,
3049 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3050 const CapturedStmt *S) {
3051 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3052 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3053 CGF.CapturedStmtInfo = &CapStmtInfo;
3054 auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3055 Fn->addFnAttr(llvm::Attribute::NoInline);
3059 void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
3060 if (!S.getAssociatedStmt()) {
3061 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3062 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3065 auto *C = S.getSingleClause<OMPSIMDClause>();
3066 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3067 PrePostActionTy &Action) {
3069 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3070 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3071 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3072 auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3073 CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars);
3077 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3080 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3081 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
3084 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
3085 QualType SrcType, QualType DestType,
3086 SourceLocation Loc) {
3087 assert(CGF.hasScalarEvaluationKind(DestType) &&
3088 "DestType must have scalar evaluation kind.");
3089 assert(!Val.isAggregate() && "Must be a scalar or complex.");
3090 return Val.isScalar()
3091 ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
3093 : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
3097 static CodeGenFunction::ComplexPairTy
3098 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
3099 QualType DestType, SourceLocation Loc) {
3100 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3101 "DestType must have complex evaluation kind.");
3102 CodeGenFunction::ComplexPairTy ComplexVal;
3103 if (Val.isScalar()) {
3104 // Convert the input element to the element type of the complex.
3105 auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3106 auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3107 DestElementType, Loc);
3108 ComplexVal = CodeGenFunction::ComplexPairTy(
3109 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3111 assert(Val.isComplex() && "Must be a scalar or complex.");
3112 auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3113 auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3114 ComplexVal.first = CGF.EmitScalarConversion(
3115 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3116 ComplexVal.second = CGF.EmitScalarConversion(
3117 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3122 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3123 LValue LVal, RValue RVal) {
3124 if (LVal.isGlobalReg()) {
3125 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3127 CGF.EmitAtomicStore(RVal, LVal,
3128 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3129 : llvm::AtomicOrdering::Monotonic,
3130 LVal.isVolatile(), /*IsInit=*/false);
3134 void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3135 QualType RValTy, SourceLocation Loc) {
3136 switch (getEvaluationKind(LVal.getType())) {
3138 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3139 *this, RVal, RValTy, LVal.getType(), Loc)),
3144 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3148 llvm_unreachable("Must be a scalar or complex.");
3152 static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3153 const Expr *X, const Expr *V,
3154 SourceLocation Loc) {
3156 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3157 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3158 LValue XLValue = CGF.EmitLValue(X);
3159 LValue VLValue = CGF.EmitLValue(V);
3160 RValue Res = XLValue.isGlobalReg()
3161 ? CGF.EmitLoadOfLValue(XLValue, Loc)
3162 : CGF.EmitAtomicLoad(
3164 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3165 : llvm::AtomicOrdering::Monotonic,
3166 XLValue.isVolatile());
3167 // OpenMP, 2.12.6, atomic Construct
3168 // Any atomic construct with a seq_cst clause forces the atomically
3169 // performed operation to include an implicit flush operation without a
3172 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3173 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3176 static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3177 const Expr *X, const Expr *E,
3178 SourceLocation Loc) {
3180 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3181 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3182 // OpenMP, 2.12.6, atomic Construct
3183 // Any atomic construct with a seq_cst clause forces the atomically
3184 // performed operation to include an implicit flush operation without a
3187 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3190 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3192 BinaryOperatorKind BO,
3193 llvm::AtomicOrdering AO,
3194 bool IsXLHSInRHSPart) {
3195 auto &Context = CGF.CGM.getContext();
3196 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3197 // expression is simple and atomic is allowed for the given type for the
3199 if (BO == BO_Comma || !Update.isScalar() ||
3200 !Update.getScalarVal()->getType()->isIntegerTy() ||
3201 !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3202 (Update.getScalarVal()->getType() !=
3203 X.getAddress().getElementType())) ||
3204 !X.getAddress().getElementType()->isIntegerTy() ||
3205 !Context.getTargetInfo().hasBuiltinAtomic(
3206 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3207 return std::make_pair(false, RValue::get(nullptr));
3209 llvm::AtomicRMWInst::BinOp RMWOp;
3212 RMWOp = llvm::AtomicRMWInst::Add;
3215 if (!IsXLHSInRHSPart)
3216 return std::make_pair(false, RValue::get(nullptr));
3217 RMWOp = llvm::AtomicRMWInst::Sub;
3220 RMWOp = llvm::AtomicRMWInst::And;
3223 RMWOp = llvm::AtomicRMWInst::Or;
3226 RMWOp = llvm::AtomicRMWInst::Xor;
3229 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3230 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3231 : llvm::AtomicRMWInst::Max)
3232 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3233 : llvm::AtomicRMWInst::UMax);
3236 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3237 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3238 : llvm::AtomicRMWInst::Min)
3239 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3240 : llvm::AtomicRMWInst::UMin);
3243 RMWOp = llvm::AtomicRMWInst::Xchg;
3252 return std::make_pair(false, RValue::get(nullptr));
3270 llvm_unreachable("Unsupported atomic update operation");
3272 auto *UpdateVal = Update.getScalarVal();
3273 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3274 UpdateVal = CGF.Builder.CreateIntCast(
3275 IC, X.getAddress().getElementType(),
3276 X.getType()->hasSignedIntegerRepresentation());
3278 auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3279 return std::make_pair(true, RValue::get(Res));
3282 std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
3283 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3284 llvm::AtomicOrdering AO, SourceLocation Loc,
3285 const llvm::function_ref<RValue(RValue)> &CommonGen) {
3286 // Update expressions are allowed to have the following forms:
3287 // x binop= expr; -> xrval + expr;
3288 // x++, ++x -> xrval + 1;
3289 // x--, --x -> xrval - 1;
3290 // x = x binop expr; -> xrval binop expr
3291 // x = expr Op x; - > expr binop xrval;
3292 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3294 if (X.isGlobalReg()) {
3295 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3297 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3299 // Perform compare-and-swap procedure.
3300 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3306 static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3307 const Expr *X, const Expr *E,
3308 const Expr *UE, bool IsXLHSInRHSPart,
3309 SourceLocation Loc) {
3310 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3311 "Update expr in 'atomic update' must be a binary operator.");
3312 auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3313 // Update expressions are allowed to have the following forms:
3314 // x binop= expr; -> xrval + expr;
3315 // x++, ++x -> xrval + 1;
3316 // x--, --x -> xrval - 1;
3317 // x = x binop expr; -> xrval binop expr
3318 // x = expr Op x; - > expr binop xrval;
3319 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3320 LValue XLValue = CGF.EmitLValue(X);
3321 RValue ExprRValue = CGF.EmitAnyExpr(E);
3322 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3323 : llvm::AtomicOrdering::Monotonic;
3324 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3325 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3326 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3327 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3329 [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
3330 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3331 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3332 return CGF.EmitAnyExpr(UE);
3334 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3335 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3336 // OpenMP, 2.12.6, atomic Construct
3337 // Any atomic construct with a seq_cst clause forces the atomically
3338 // performed operation to include an implicit flush operation without a
3341 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3344 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
3345 QualType SourceType, QualType ResType,
3346 SourceLocation Loc) {
3347 switch (CGF.getEvaluationKind(ResType)) {
3350 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3352 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3353 return RValue::getComplex(Res.first, Res.second);
3358 llvm_unreachable("Must be a scalar or complex.");
3361 static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3362 bool IsPostfixUpdate, const Expr *V,
3363 const Expr *X, const Expr *E,
3364 const Expr *UE, bool IsXLHSInRHSPart,
3365 SourceLocation Loc) {
3366 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3367 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3369 LValue VLValue = CGF.EmitLValue(V);
3370 LValue XLValue = CGF.EmitLValue(X);
3371 RValue ExprRValue = CGF.EmitAnyExpr(E);
3372 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3373 : llvm::AtomicOrdering::Monotonic;
3374 QualType NewVValType;
3376 // 'x' is updated with some additional value.
3377 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3378 "Update expr in 'atomic capture' must be a binary operator.");
3379 auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3380 // Update expressions are allowed to have the following forms:
3381 // x binop= expr; -> xrval + expr;
3382 // x++, ++x -> xrval + 1;
3383 // x--, --x -> xrval - 1;
3384 // x = x binop expr; -> xrval binop expr
3385 // x = expr Op x; - > expr binop xrval;
3386 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3387 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3388 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3389 NewVValType = XRValExpr->getType();
3390 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3391 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3392 IsPostfixUpdate](RValue XRValue) -> RValue {
3393 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3394 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3395 RValue Res = CGF.EmitAnyExpr(UE);
3396 NewVVal = IsPostfixUpdate ? XRValue : Res;
3399 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3400 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3402 // 'atomicrmw' instruction was generated.
3403 if (IsPostfixUpdate) {
3404 // Use old value from 'atomicrmw'.
3405 NewVVal = Res.second;
3407 // 'atomicrmw' does not provide new value, so evaluate it using old
3409 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3410 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3411 NewVVal = CGF.EmitAnyExpr(UE);
3415 // 'x' is simply rewritten with some 'expr'.
3416 NewVValType = X->getType().getNonReferenceType();
3417 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3418 X->getType().getNonReferenceType(), Loc);
3419 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) -> RValue {
3423 // Try to perform atomicrmw xchg, otherwise simple exchange.
3424 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3425 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3428 // 'atomicrmw' instruction was generated.
3429 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3432 // Emit post-update store to 'v' of old/new 'x' value.
3433 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3434 // OpenMP, 2.12.6, atomic Construct
3435 // Any atomic construct with a seq_cst clause forces the atomically
3436 // performed operation to include an implicit flush operation without a
3439 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3442 static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
3443 bool IsSeqCst, bool IsPostfixUpdate,
3444 const Expr *X, const Expr *V, const Expr *E,
3445 const Expr *UE, bool IsXLHSInRHSPart,
3446 SourceLocation Loc) {
3449 EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3452 EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3456 EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3459 EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3460 IsXLHSInRHSPart, Loc);
3464 case OMPC_num_threads:
3466 case OMPC_firstprivate:
3467 case OMPC_lastprivate:
3468 case OMPC_reduction:
3469 case OMPC_task_reduction:
3479 case OMPC_copyprivate:
3481 case OMPC_proc_bind:
3486 case OMPC_threadprivate:
3488 case OMPC_mergeable:
3493 case OMPC_num_teams:
3494 case OMPC_thread_limit:
3496 case OMPC_grainsize:
3498 case OMPC_num_tasks:
3500 case OMPC_dist_schedule:
3501 case OMPC_defaultmap:
3505 case OMPC_use_device_ptr:
3506 case OMPC_is_device_ptr:
3507 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3511 void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
3512 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3513 OpenMPClauseKind Kind = OMPC_unknown;
3514 for (auto *C : S.clauses()) {
3515 // Find first clause (skip seq_cst clause, if it is first).
3516 if (C->getClauseKind() != OMPC_seq_cst) {
3517 Kind = C->getClauseKind();
3523 S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
3524 if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
3525 enterFullExpression(EWC);
3527 // Processing for statements under 'atomic capture'.
3528 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3529 for (const auto *C : Compound->body()) {
3530 if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3531 enterFullExpression(EWC);
3536 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3537 PrePostActionTy &) {
3538 CGF.EmitStopPoint(CS);
3539 EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3540 S.getV(), S.getExpr(), S.getUpdateExpr(),
3541 S.isXLHSInRHSPart(), S.getLocStart());
3543 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3544 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3547 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
3548 const OMPExecutableDirective &S,
3549 const RegionCodeGenTy &CodeGen) {
3550 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
3551 CodeGenModule &CGM = CGF.CGM;
3552 const CapturedStmt &CS = *cast<CapturedStmt>(S.getAssociatedStmt());
3554 llvm::Function *Fn = nullptr;
3555 llvm::Constant *FnID = nullptr;
3557 const Expr *IfCond = nullptr;
3558 // Check for the at most one if clause associated with the target region.
3559 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3560 if (C->getNameModifier() == OMPD_unknown ||
3561 C->getNameModifier() == OMPD_target) {
3562 IfCond = C->getCondition();
3567 // Check if we have any device clause associated with the directive.
3568 const Expr *Device = nullptr;
3569 if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
3570 Device = C->getDevice();
3573 // Check if we have an if clause whose conditional always evaluates to false
3574 // or if we do not have any targets specified. If so the target region is not
3575 // an offload entry point.
3576 bool IsOffloadEntry = true;
3579 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3580 IsOffloadEntry = false;
3582 if (CGM.getLangOpts().OMPTargetTriples.empty())
3583 IsOffloadEntry = false;
3585 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3586 StringRef ParentName;
3587 // In case we have Ctors/Dtors we use the complete type variant to produce
3588 // the mangling of the device outlined kernel.
3589 if (auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3590 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3591 else if (auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3592 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3595 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
3597 // Emit target region as a standalone region.
3598 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
3599 IsOffloadEntry, CodeGen);
3600 OMPLexicalScope Scope(CGF, S);
3601 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3602 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
3603 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
3607 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
3608 PrePostActionTy &Action) {
3609 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3610 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3611 CGF.EmitOMPPrivateClause(S, PrivateScope);
3612 (void)PrivateScope.Privatize();
3615 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3618 void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3619 StringRef ParentName,
3620 const OMPTargetDirective &S) {
3621 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3622 emitTargetRegion(CGF, S, Action);
3625 llvm::Constant *Addr;
3626 // Emit target region as a standalone region.
3627 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3628 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3629 assert(Fn && Addr && "Target device function emission failed.");
3632 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
3633 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3634 emitTargetRegion(CGF, S, Action);
3636 emitCommonOMPTargetDirective(*this, S, CodeGen);
3639 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
3640 const OMPExecutableDirective &S,
3641 OpenMPDirectiveKind InnermostKind,
3642 const RegionCodeGenTy &CodeGen) {
3643 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
3644 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
3645 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
3647 const OMPNumTeamsClause *NT = S.getSingleClause<OMPNumTeamsClause>();
3648 const OMPThreadLimitClause *TL = S.getSingleClause<OMPThreadLimitClause>();
3650 Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr;
3651 Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr;
3653 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
3657 OMPTeamsScope Scope(CGF, S);
3658 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3659 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3660 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
3664 void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
3665 // Emit teams region as a standalone region.
3666 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3667 OMPPrivateScope PrivateScope(CGF);
3668 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3669 CGF.EmitOMPPrivateClause(S, PrivateScope);
3670 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3671 (void)PrivateScope.Privatize();
3672 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3673 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3675 emitCommonOMPTeamsDirective(*this, S, OMPD_teams, CodeGen);
3676 emitPostUpdateForReductionClause(
3677 *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
3680 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
3681 const OMPTargetTeamsDirective &S) {
3682 auto *CS = S.getCapturedStmt(OMPD_teams);
3684 auto &&CodeGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3685 // TODO: Add support for clauses.
3686 CGF.EmitStmt(CS->getCapturedStmt());
3688 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
3691 void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
3692 CodeGenModule &CGM, StringRef ParentName,
3693 const OMPTargetTeamsDirective &S) {
3694 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3695 emitTargetTeamsRegion(CGF, Action, S);
3698 llvm::Constant *Addr;
3699 // Emit target region as a standalone region.
3700 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3701 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3702 assert(Fn && Addr && "Target device function emission failed.");
3705 void CodeGenFunction::EmitOMPTargetTeamsDirective(
3706 const OMPTargetTeamsDirective &S) {
3707 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3708 emitTargetTeamsRegion(CGF, Action, S);
3710 emitCommonOMPTargetDirective(*this, S, CodeGen);
3713 void CodeGenFunction::EmitOMPCancellationPointDirective(
3714 const OMPCancellationPointDirective &S) {
3715 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
3716 S.getCancelRegion());
3719 void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
3720 const Expr *IfCond = nullptr;
3721 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3722 if (C->getNameModifier() == OMPD_unknown ||
3723 C->getNameModifier() == OMPD_cancel) {
3724 IfCond = C->getCondition();
3728 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
3729 S.getCancelRegion());
3732 CodeGenFunction::JumpDest
3733 CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
3734 if (Kind == OMPD_parallel || Kind == OMPD_task ||
3735 Kind == OMPD_target_parallel)
3737 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
3738 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
3739 Kind == OMPD_distribute_parallel_for ||
3740 Kind == OMPD_target_parallel_for);
3741 return OMPCancelStack.getExitBlock();
3744 void CodeGenFunction::EmitOMPUseDevicePtrClause(
3745 const OMPClause &NC, OMPPrivateScope &PrivateScope,
3746 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
3747 const auto &C = cast<OMPUseDevicePtrClause>(NC);
3748 auto OrigVarIt = C.varlist_begin();
3749 auto InitIt = C.inits().begin();
3750 for (auto PvtVarIt : C.private_copies()) {
3751 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
3752 auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
3753 auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
3755 // In order to identify the right initializer we need to match the
3756 // declaration used by the mapping logic. In some cases we may get
3757 // OMPCapturedExprDecl that refers to the original declaration.
3758 const ValueDecl *MatchingVD = OrigVD;
3759 if (auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
3760 // OMPCapturedExprDecl are used to privative fields of the current
3762 auto *ME = cast<MemberExpr>(OED->getInit());
3763 assert(isa<CXXThisExpr>(ME->getBase()) &&
3764 "Base should be the current struct!");
3765 MatchingVD = ME->getMemberDecl();
3768 // If we don't have information about the current list item, move on to
3770 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
3771 if (InitAddrIt == CaptureDeviceAddrMap.end())
3774 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
3775 // Initialize the temporary initialization variable with the address we
3776 // get from the runtime library. We have to cast the source address
3777 // because it is always a void *. References are materialized in the
3778 // privatization scope, so the initialization here disregards the fact
3779 // the original variable is a reference.
3781 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
3782 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
3783 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
3784 setAddrOfLocalVar(InitVD, InitAddr);
3786 // Emit private declaration, it will be initialized by the value we
3787 // declaration we just added to the local declarations map.
3790 // The initialization variables reached its purpose in the emission
3791 // ofthe previous declaration, so we don't need it anymore.
3792 LocalDeclMap.erase(InitVD);
3794 // Return the address of the private variable.
3795 return GetAddrOfLocalVar(PvtVD);
3797 assert(IsRegistered && "firstprivate var already registered as private");
3798 // Silence the warning about unused variable.
3806 // Generate the instructions for '#pragma omp target data' directive.
3807 void CodeGenFunction::EmitOMPTargetDataDirective(
3808 const OMPTargetDataDirective &S) {
3809 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
3811 // Create a pre/post action to signal the privatization of the device pointer.
3812 // This action can be replaced by the OpenMP runtime code generation to
3813 // deactivate privatization.
3814 bool PrivatizeDevicePointers = false;
3815 class DevicePointerPrivActionTy : public PrePostActionTy {
3816 bool &PrivatizeDevicePointers;
3819 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
3820 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
3821 void Enter(CodeGenFunction &CGF) override {
3822 PrivatizeDevicePointers = true;
3825 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
3827 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
3828 CodeGenFunction &CGF, PrePostActionTy &Action) {
3829 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3831 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3834 // Codegen that selects wheather to generate the privatization code or not.
3835 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
3836 &InnermostCodeGen](CodeGenFunction &CGF,
3837 PrePostActionTy &Action) {
3838 RegionCodeGenTy RCG(InnermostCodeGen);
3839 PrivatizeDevicePointers = false;
3841 // Call the pre-action to change the status of PrivatizeDevicePointers if
3845 if (PrivatizeDevicePointers) {
3846 OMPPrivateScope PrivateScope(CGF);
3847 // Emit all instances of the use_device_ptr clause.
3848 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
3849 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
3850 Info.CaptureDeviceAddrMap);
3851 (void)PrivateScope.Privatize();
3857 // Forward the provided action to the privatization codegen.
3858 RegionCodeGenTy PrivRCG(PrivCodeGen);
3859 PrivRCG.setAction(Action);
3861 // Notwithstanding the body of the region is emitted as inlined directive,
3862 // we don't use an inline scope as changes in the references inside the
3863 // region are expected to be visible outside, so we do not privative them.
3864 OMPLexicalScope Scope(CGF, S);
3865 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
3869 RegionCodeGenTy RCG(CodeGen);
3871 // If we don't have target devices, don't bother emitting the data mapping
3873 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
3878 // Check if we have any if clause associated with the directive.
3879 const Expr *IfCond = nullptr;
3880 if (auto *C = S.getSingleClause<OMPIfClause>())
3881 IfCond = C->getCondition();
3883 // Check if we have any device clause associated with the directive.
3884 const Expr *Device = nullptr;
3885 if (auto *C = S.getSingleClause<OMPDeviceClause>())
3886 Device = C->getDevice();
3888 // Set the action to signal privatization of device pointers.
3889 RCG.setAction(PrivAction);
3891 // Emit region code.
3892 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
3896 void CodeGenFunction::EmitOMPTargetEnterDataDirective(
3897 const OMPTargetEnterDataDirective &S) {
3898 // If we don't have target devices, don't bother emitting the data mapping
3900 if (CGM.getLangOpts().OMPTargetTriples.empty())
3903 // Check if we have any if clause associated with the directive.
3904 const Expr *IfCond = nullptr;
3905 if (auto *C = S.getSingleClause<OMPIfClause>())
3906 IfCond = C->getCondition();
3908 // Check if we have any device clause associated with the directive.
3909 const Expr *Device = nullptr;
3910 if (auto *C = S.getSingleClause<OMPDeviceClause>())
3911 Device = C->getDevice();
3913 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
3916 void CodeGenFunction::EmitOMPTargetExitDataDirective(
3917 const OMPTargetExitDataDirective &S) {
3918 // If we don't have target devices, don't bother emitting the data mapping
3920 if (CGM.getLangOpts().OMPTargetTriples.empty())
3923 // Check if we have any if clause associated with the directive.
3924 const Expr *IfCond = nullptr;
3925 if (auto *C = S.getSingleClause<OMPIfClause>())
3926 IfCond = C->getCondition();
3928 // Check if we have any device clause associated with the directive.
3929 const Expr *Device = nullptr;
3930 if (auto *C = S.getSingleClause<OMPDeviceClause>())
3931 Device = C->getDevice();
3933 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
3936 static void emitTargetParallelRegion(CodeGenFunction &CGF,
3937 const OMPTargetParallelDirective &S,
3938 PrePostActionTy &Action) {
3939 // Get the captured statement associated with the 'parallel' region.
3940 auto *CS = S.getCapturedStmt(OMPD_parallel);
3942 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) {
3943 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3944 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3945 CGF.EmitOMPPrivateClause(S, PrivateScope);
3946 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3947 (void)PrivateScope.Privatize();
3948 // TODO: Add support for clauses.
3949 CGF.EmitStmt(CS->getCapturedStmt());
3950 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
3952 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
3953 emitEmptyBoundParameters);
3954 emitPostUpdateForReductionClause(
3955 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
3958 void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
3959 CodeGenModule &CGM, StringRef ParentName,
3960 const OMPTargetParallelDirective &S) {
3961 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3962 emitTargetParallelRegion(CGF, S, Action);
3965 llvm::Constant *Addr;
3966 // Emit target region as a standalone region.
3967 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3968 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3969 assert(Fn && Addr && "Target device function emission failed.");
3972 void CodeGenFunction::EmitOMPTargetParallelDirective(
3973 const OMPTargetParallelDirective &S) {
3974 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3975 emitTargetParallelRegion(CGF, S, Action);
3977 emitCommonOMPTargetDirective(*this, S, CodeGen);
3980 void CodeGenFunction::EmitOMPTargetParallelForDirective(
3981 const OMPTargetParallelForDirective &S) {
3982 // TODO: codegen for target parallel for.
3985 /// Emit a helper variable and return corresponding lvalue.
3986 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
3987 const ImplicitParamDecl *PVD,
3988 CodeGenFunction::OMPPrivateScope &Privates) {
3989 auto *VDecl = cast<VarDecl>(Helper->getDecl());
3990 Privates.addPrivate(
3991 VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); });
3994 void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
3995 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
3996 // Emit outlined function for task construct.
3997 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3998 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3999 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4000 const Expr *IfCond = nullptr;
4001 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4002 if (C->getNameModifier() == OMPD_unknown ||
4003 C->getNameModifier() == OMPD_taskloop) {
4004 IfCond = C->getCondition();
4010 // Check if taskloop must be emitted without taskgroup.
4011 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
4012 // TODO: Check if we should emit tied or untied task.
4014 // Set scheduling for taskloop
4015 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4017 Data.Schedule.setInt(/*IntVal=*/false);
4018 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4019 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4021 Data.Schedule.setInt(/*IntVal=*/true);
4022 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4025 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4027 // for (IV in 0..LastIteration) BODY;
4028 // <Final counter/linear vars updates>;
4032 // Emit: if (PreCond) - begin.
4033 // If the condition constant folds and can be elided, avoid emitting the
4036 llvm::BasicBlock *ContBlock = nullptr;
4037 OMPLoopScope PreInitScope(CGF, S);
4038 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4042 auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4043 ContBlock = CGF.createBasicBlock("taskloop.if.end");
4044 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4045 CGF.getProfileCount(&S));
4046 CGF.EmitBlock(ThenBlock);
4047 CGF.incrementProfileCounter(&S);
4050 if (isOpenMPSimdDirective(S.getDirectiveKind()))
4051 CGF.EmitOMPSimdInit(S);
4053 OMPPrivateScope LoopScope(CGF);
4054 // Emit helper vars inits.
4055 enum { LowerBound = 5, UpperBound, Stride, LastIter };
4056 auto *I = CS->getCapturedDecl()->param_begin();
4057 auto *LBP = std::next(I, LowerBound);
4058 auto *UBP = std::next(I, UpperBound);
4059 auto *STP = std::next(I, Stride);
4060 auto *LIP = std::next(I, LastIter);
4061 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4063 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4065 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4066 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4068 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4069 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4070 (void)LoopScope.Privatize();
4071 // Emit the loop iteration variable.
4072 const Expr *IVExpr = S.getIterationVariable();
4073 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4074 CGF.EmitVarDecl(*IVDecl);
4075 CGF.EmitIgnoredExpr(S.getInit());
4077 // Emit the iterations count variable.
4078 // If it is not a variable, Sema decided to calculate iterations count on
4079 // each iteration (e.g., it is foldable into a constant).
4080 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4081 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4082 // Emit calculation of the iterations count.
4083 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4086 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4088 [&S](CodeGenFunction &CGF) {
4089 CGF.EmitOMPLoopBody(S, JumpDest());
4090 CGF.EmitStopPoint(&S);
4092 [](CodeGenFunction &) {});
4093 // Emit: if (PreCond) - end.
4095 CGF.EmitBranch(ContBlock);
4096 CGF.EmitBlock(ContBlock, true);
4098 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4099 if (HasLastprivateClause) {
4100 CGF.EmitOMPLastprivateClauseFinal(
4101 S, isOpenMPSimdDirective(S.getDirectiveKind()),
4102 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4103 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4104 (*LIP)->getType(), S.getLocStart())));
4107 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4108 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4109 const OMPTaskDataTy &Data) {
4110 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) {
4111 OMPLoopScope PreInitScope(CGF, S);
4112 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4113 OutlinedFn, SharedsTy,
4114 CapturedStruct, IfCond, Data);
4116 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4120 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4122 CGM.getOpenMPRuntime().emitTaskgroupRegion(
4124 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4125 PrePostActionTy &Action) {
4127 CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4133 void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
4134 EmitOMPTaskLoopBasedDirective(S);
4137 void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
4138 const OMPTaskLoopSimdDirective &S) {
4139 EmitOMPTaskLoopBasedDirective(S);
4142 // Generate the instructions for '#pragma omp target update' directive.
4143 void CodeGenFunction::EmitOMPTargetUpdateDirective(
4144 const OMPTargetUpdateDirective &S) {
4145 // If we don't have target devices, don't bother emitting the data mapping
4147 if (CGM.getLangOpts().OMPTargetTriples.empty())
4150 // Check if we have any if clause associated with the directive.
4151 const Expr *IfCond = nullptr;
4152 if (auto *C = S.getSingleClause<OMPIfClause>())
4153 IfCond = C->getCondition();
4155 // Check if we have any device clause associated with the directive.
4156 const Expr *Device = nullptr;
4157 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4158 Device = C->getDevice();
4160 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);