1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This contains code to emit OpenMP nodes as LLVM code.
12 //===----------------------------------------------------------------------===//
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31 for (const auto *C : S.clauses()) {
32 if (auto *CPI = OMPClauseWithPreInit::get(C)) {
33 if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34 for (const auto *I : PreInit->decls()) {
35 if (!I->hasAttr<OMPCaptureNoInitAttr>())
36 CGF.EmitVarDecl(cast<VarDecl>(*I));
38 CodeGenFunction::AutoVarEmission Emission =
39 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40 CGF.EmitAutoVarCleanups(Emission);
47 CodeGenFunction::OMPPrivateScope InlinedShareds;
49 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50 return CGF.LambdaCaptureFields.lookup(VD) ||
51 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
56 OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S,
57 bool AsInlined = false, bool EmitPreInitStmt = true)
58 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
61 emitPreInitStmt(CGF, S);
63 if (S.hasAssociatedStmt()) {
64 auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
65 for (auto &C : CS->captures()) {
66 if (C.capturesVariable() || C.capturesVariableByCopy()) {
67 auto *VD = C.getCapturedVar();
68 assert(VD == VD->getCanonicalDecl() &&
69 "Canonical decl must be captured.");
70 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
71 isCapturedVar(CGF, VD) ||
72 (CGF.CapturedStmtInfo &&
73 InlinedShareds.isGlobalVarCaptured(VD)),
74 VD->getType().getNonReferenceType(), VK_LValue,
76 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
77 return CGF.EmitLValue(&DRE).getAddress();
81 (void)InlinedShareds.Privatize();
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
91 OpenMPDirectiveKind Kind = S.getDirectiveKind();
92 return !(isOpenMPTargetExecutionDirective(Kind) ||
93 isOpenMPLoopBoundSharingDirective(Kind)) &&
94 isOpenMPParallelDirective(Kind);
98 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99 : OMPLexicalScope(CGF, S,
101 /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
104 /// Lexical scope for OpenMP teams construct, that handles correct codegen
105 /// for captured expressions.
106 class OMPTeamsScope final : public OMPLexicalScope {
107 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108 OpenMPDirectiveKind Kind = S.getDirectiveKind();
109 return !isOpenMPTargetExecutionDirective(Kind) &&
110 isOpenMPTeamsDirective(Kind);
114 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
115 : OMPLexicalScope(CGF, S,
117 /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
125 for (auto *E : S.counters()) {
126 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127 (void)PreCondScope.addPrivate(VD, [&CGF, VD]() {
128 return CGF.CreateMemTemp(VD->getType().getNonReferenceType());
131 (void)PreCondScope.Privatize();
132 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
133 if (auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
134 for (const auto *I : PreInits->decls())
135 CGF.EmitVarDecl(cast<VarDecl>(*I));
141 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
142 : CodeGenFunction::RunCleanupsScope(CGF) {
143 emitPreInitStmt(CGF, S);
147 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
148 CodeGenFunction::OMPPrivateScope InlinedShareds;
150 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
151 return CGF.LambdaCaptureFields.lookup(VD) ||
152 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
153 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
154 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
158 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
159 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
160 InlinedShareds(CGF) {
161 for (const auto *C : S.clauses()) {
162 if (auto *CPI = OMPClauseWithPreInit::get(C)) {
163 if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
164 for (const auto *I : PreInit->decls()) {
165 if (!I->hasAttr<OMPCaptureNoInitAttr>())
166 CGF.EmitVarDecl(cast<VarDecl>(*I));
168 CodeGenFunction::AutoVarEmission Emission =
169 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
170 CGF.EmitAutoVarCleanups(Emission);
174 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
175 for (const Expr *E : UDP->varlists()) {
176 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
177 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
178 CGF.EmitVarDecl(*OED);
182 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
183 CGF.EmitOMPPrivateClause(S, InlinedShareds);
184 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
185 if (const Expr *E = TG->getReductionRef())
186 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
188 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
190 for (auto &C : CS->captures()) {
191 if (C.capturesVariable() || C.capturesVariableByCopy()) {
192 auto *VD = C.getCapturedVar();
193 assert(VD == VD->getCanonicalDecl() &&
194 "Canonical decl must be captured.");
195 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
196 isCapturedVar(CGF, VD) ||
197 (CGF.CapturedStmtInfo &&
198 InlinedShareds.isGlobalVarCaptured(VD)),
199 VD->getType().getNonReferenceType(), VK_LValue,
201 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
202 return CGF.EmitLValue(&DRE).getAddress();
206 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
208 (void)InlinedShareds.Privatize();
214 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
215 const OMPExecutableDirective &S,
216 const RegionCodeGenTy &CodeGen);
218 LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
219 if (auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
220 if (auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
221 OrigVD = OrigVD->getCanonicalDecl();
223 LambdaCaptureFields.lookup(OrigVD) ||
224 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
225 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
226 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
227 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
228 return EmitLValue(&DRE);
231 return EmitLValue(E);
234 llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
235 auto &C = getContext();
236 llvm::Value *Size = nullptr;
237 auto SizeInChars = C.getTypeSizeInChars(Ty);
238 if (SizeInChars.isZero()) {
239 // getTypeSizeInChars() returns 0 for a VLA.
240 while (auto *VAT = C.getAsVariableArrayType(Ty)) {
241 llvm::Value *ArraySize;
242 std::tie(ArraySize, Ty) = getVLASize(VAT);
243 Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
245 SizeInChars = C.getTypeSizeInChars(Ty);
246 if (SizeInChars.isZero())
247 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248 Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
250 Size = CGM.getSize(SizeInChars);
254 void CodeGenFunction::GenerateOpenMPCapturedVars(
255 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
256 const RecordDecl *RD = S.getCapturedRecordDecl();
257 auto CurField = RD->field_begin();
258 auto CurCap = S.captures().begin();
259 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
260 E = S.capture_init_end();
261 I != E; ++I, ++CurField, ++CurCap) {
262 if (CurField->hasCapturedVLAType()) {
263 auto VAT = CurField->getCapturedVLAType();
264 auto *Val = VLASizeMap[VAT->getSizeExpr()];
265 CapturedVars.push_back(Val);
266 } else if (CurCap->capturesThis())
267 CapturedVars.push_back(CXXThisValue);
268 else if (CurCap->capturesVariableByCopy()) {
270 EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal();
272 // If the field is not a pointer, we need to save the actual value
273 // and load it as a void pointer.
274 if (!CurField->getType()->isAnyPointerType()) {
275 auto &Ctx = getContext();
276 auto DstAddr = CreateMemTemp(
277 Ctx.getUIntPtrType(),
278 Twine(CurCap->getCapturedVar()->getName()) + ".casted");
279 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
281 auto *SrcAddrVal = EmitScalarConversion(
282 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
283 Ctx.getPointerType(CurField->getType()), SourceLocation());
285 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
287 // Store the value using the source type pointer.
288 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
290 // Load the value using the destination type pointer.
291 CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
293 CapturedVars.push_back(CV);
295 assert(CurCap->capturesVariable() && "Expected capture by reference.");
296 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
301 static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType,
302 StringRef Name, LValue AddrLV,
303 bool isReferenceType = false) {
304 ASTContext &Ctx = CGF.getContext();
306 auto *CastedPtr = CGF.EmitScalarConversion(
307 AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
308 Ctx.getPointerType(DstType), SourceLocation());
310 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
313 // If we are dealing with references we need to return the address of the
314 // reference instead of the reference of the value.
315 if (isReferenceType) {
316 QualType RefType = Ctx.getLValueReferenceType(DstType);
317 auto *RefVal = TmpAddr.getPointer();
318 TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
319 auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
320 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true);
326 static QualType getCanonicalParamType(ASTContext &C, QualType T) {
327 if (T->isLValueReferenceType()) {
328 return C.getLValueReferenceType(
329 getCanonicalParamType(C, T.getNonReferenceType()),
330 /*SpelledAsLValue=*/false);
332 if (T->isPointerType())
333 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
334 if (auto *A = T->getAsArrayTypeUnsafe()) {
335 if (auto *VLA = dyn_cast<VariableArrayType>(A))
336 return getCanonicalParamType(C, VLA->getElementType());
337 else if (!A->isVariablyModifiedType())
338 return C.getCanonicalType(T);
340 return C.getCanonicalParamType(T);
344 /// Contains required data for proper outlined function codegen.
345 struct FunctionOptions {
346 /// Captured statement for which the function is generated.
347 const CapturedStmt *S = nullptr;
348 /// true if cast to/from UIntPtr is required for variables captured by
350 const bool UIntPtrCastRequired = true;
351 /// true if only casted arguments must be registered as local args or VLA
353 const bool RegisterCastedArgsOnly = false;
354 /// Name of the generated function.
355 const StringRef FunctionName;
356 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
357 bool RegisterCastedArgsOnly,
358 StringRef FunctionName)
359 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
360 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
361 FunctionName(FunctionName) {}
365 static llvm::Function *emitOutlinedFunctionPrologue(
366 CodeGenFunction &CGF, FunctionArgList &Args,
367 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
369 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
371 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
372 const CapturedDecl *CD = FO.S->getCapturedDecl();
373 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
374 assert(CD->hasBody() && "missing CapturedDecl body");
376 CXXThisValue = nullptr;
377 // Build the argument list.
378 CodeGenModule &CGM = CGF.CGM;
379 ASTContext &Ctx = CGM.getContext();
380 FunctionArgList TargetArgs;
381 Args.append(CD->param_begin(),
382 std::next(CD->param_begin(), CD->getContextParamPosition()));
385 std::next(CD->param_begin(), CD->getContextParamPosition()));
386 auto I = FO.S->captures().begin();
387 FunctionDecl *DebugFunctionDecl = nullptr;
388 if (!FO.UIntPtrCastRequired) {
389 FunctionProtoType::ExtProtoInfo EPI;
390 DebugFunctionDecl = FunctionDecl::Create(
391 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(),
392 SourceLocation(), DeclarationName(), Ctx.VoidTy,
393 Ctx.getTrivialTypeSourceInfo(
394 Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
395 SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
397 for (auto *FD : RD->fields()) {
398 QualType ArgType = FD->getType();
399 IdentifierInfo *II = nullptr;
400 VarDecl *CapVar = nullptr;
402 // If this is a capture by copy and the type is not a pointer, the outlined
403 // function argument type should be uintptr and the value properly casted to
404 // uintptr. This is necessary given that the runtime library is only able to
405 // deal with pointers. We can pass in the same way the VLA type sizes to the
406 // outlined function.
407 if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
408 I->capturesVariableArrayType()) {
409 if (FO.UIntPtrCastRequired)
410 ArgType = Ctx.getUIntPtrType();
413 if (I->capturesVariable() || I->capturesVariableByCopy()) {
414 CapVar = I->getCapturedVar();
415 II = CapVar->getIdentifier();
416 } else if (I->capturesThis())
417 II = &Ctx.Idents.get("this");
419 assert(I->capturesVariableArrayType());
420 II = &Ctx.Idents.get("vla");
422 if (ArgType->isVariablyModifiedType())
423 ArgType = getCanonicalParamType(Ctx, ArgType);
425 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
426 Arg = ParmVarDecl::Create(
427 Ctx, DebugFunctionDecl,
428 CapVar ? CapVar->getLocStart() : FD->getLocStart(),
429 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
430 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
432 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
433 II, ArgType, ImplicitParamDecl::Other);
435 Args.emplace_back(Arg);
436 // Do not cast arguments if we emit function with non-original types.
437 TargetArgs.emplace_back(
438 FO.UIntPtrCastRequired
440 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
444 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
447 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
450 // Create the function declaration.
451 const CGFunctionInfo &FuncInfo =
452 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
453 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
456 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
457 FO.FunctionName, &CGM.getModule());
458 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
460 F->setDoesNotThrow();
462 // Generate the function.
463 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
464 FO.S->getLocStart(), CD->getBody()->getLocStart());
465 unsigned Cnt = CD->getContextParamPosition();
466 I = FO.S->captures().begin();
467 for (auto *FD : RD->fields()) {
468 // Do not map arguments if we emit function with non-original types.
469 Address LocalAddr(Address::invalid());
470 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
471 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
474 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
476 // If we are capturing a pointer by copy we don't need to do anything, just
477 // use the value that we get from the arguments.
478 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
479 const VarDecl *CurVD = I->getCapturedVar();
480 // If the variable is a reference we need to materialize it here.
481 if (CurVD->getType()->isReferenceType()) {
482 Address RefAddr = CGF.CreateMemTemp(
483 CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
484 CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
485 /*Volatile=*/false, CurVD->getType());
488 if (!FO.RegisterCastedArgsOnly)
489 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
495 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
496 AlignmentSource::Decl);
497 if (FD->hasCapturedVLAType()) {
498 if (FO.UIntPtrCastRequired) {
499 ArgLVal = CGF.MakeAddrLValue(castValueFromUintptr(CGF, FD->getType(),
500 Args[Cnt]->getName(),
502 FD->getType(), AlignmentSource::Decl);
505 CGF.EmitLoadOfLValue(ArgLVal, SourceLocation()).getScalarVal();
506 auto VAT = FD->getCapturedVLAType();
507 VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}});
508 } else if (I->capturesVariable()) {
509 auto *Var = I->getCapturedVar();
510 QualType VarTy = Var->getType();
511 Address ArgAddr = ArgLVal.getAddress();
512 if (!VarTy->isReferenceType()) {
513 if (ArgLVal.getType()->isLValueReferenceType()) {
514 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
515 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
516 assert(ArgLVal.getType()->isPointerType());
517 ArgAddr = CGF.EmitLoadOfPointer(
518 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
521 if (!FO.RegisterCastedArgsOnly) {
524 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
526 } else if (I->capturesVariableByCopy()) {
527 assert(!FD->getType()->isAnyPointerType() &&
528 "Not expecting a captured pointer.");
529 auto *Var = I->getCapturedVar();
530 QualType VarTy = Var->getType();
534 FO.UIntPtrCastRequired
535 ? castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(),
536 ArgLVal, VarTy->isReferenceType())
537 : ArgLVal.getAddress()}});
539 // If 'this' is captured, load it into CXXThisValue.
540 assert(I->capturesThis());
541 CXXThisValue = CGF.EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation())
543 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
553 CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
556 "CapturedStmtInfo should be set when generating the captured function");
557 const CapturedDecl *CD = S.getCapturedDecl();
558 // Build the argument list.
559 bool NeedWrapperFunction =
561 CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
562 FunctionArgList Args;
563 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
564 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
565 SmallString<256> Buffer;
566 llvm::raw_svector_ostream Out(Buffer);
567 Out << CapturedStmtInfo->getHelperName();
568 if (NeedWrapperFunction)
570 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
572 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
573 VLASizes, CXXThisValue, FO);
574 for (const auto &LocalAddrPair : LocalAddrs) {
575 if (LocalAddrPair.second.first) {
576 setAddrOfLocalVar(LocalAddrPair.second.first,
577 LocalAddrPair.second.second);
580 for (const auto &VLASizePair : VLASizes)
581 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
582 PGO.assignRegionCounters(GlobalDecl(CD), F);
583 CapturedStmtInfo->EmitBody(*this, CD->getBody());
584 FinishFunction(CD->getBodyRBrace());
585 if (!NeedWrapperFunction)
588 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
589 /*RegisterCastedArgsOnly=*/true,
590 CapturedStmtInfo->getHelperName());
591 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
595 llvm::Function *WrapperF =
596 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
597 WrapperCGF.CXXThisValue, WrapperFO);
598 llvm::SmallVector<llvm::Value *, 4> CallArgs;
599 for (const auto *Arg : Args) {
600 llvm::Value *CallArg;
601 auto I = LocalAddrs.find(Arg);
602 if (I != LocalAddrs.end()) {
603 LValue LV = WrapperCGF.MakeAddrLValue(
605 I->second.first ? I->second.first->getType() : Arg->getType(),
606 AlignmentSource::Decl);
607 CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
609 auto EI = VLASizes.find(Arg);
610 if (EI != VLASizes.end())
611 CallArg = EI->second.second;
613 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
615 AlignmentSource::Decl);
616 CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
619 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
621 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(),
623 WrapperCGF.FinishFunction();
627 //===----------------------------------------------------------------------===//
628 // OpenMP Directive Emission
629 //===----------------------------------------------------------------------===//
630 void CodeGenFunction::EmitOMPAggregateAssign(
631 Address DestAddr, Address SrcAddr, QualType OriginalType,
632 const llvm::function_ref<void(Address, Address)> &CopyGen) {
633 // Perform element-by-element initialization.
636 // Drill down to the base element type on both arrays.
637 auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
638 auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
639 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
641 auto SrcBegin = SrcAddr.getPointer();
642 auto DestBegin = DestAddr.getPointer();
643 // Cast from pointer to array type to pointer to single element.
644 auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
645 // The basic structure here is a while-do loop.
646 auto BodyBB = createBasicBlock("omp.arraycpy.body");
647 auto DoneBB = createBasicBlock("omp.arraycpy.done");
649 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
650 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
652 // Enter the loop body, making that address the current address.
653 auto EntryBB = Builder.GetInsertBlock();
656 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
658 llvm::PHINode *SrcElementPHI =
659 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
660 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
661 Address SrcElementCurrent =
662 Address(SrcElementPHI,
663 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
665 llvm::PHINode *DestElementPHI =
666 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
667 DestElementPHI->addIncoming(DestBegin, EntryBB);
668 Address DestElementCurrent =
669 Address(DestElementPHI,
670 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
673 CopyGen(DestElementCurrent, SrcElementCurrent);
675 // Shift the address forward by one element.
676 auto DestElementNext = Builder.CreateConstGEP1_32(
677 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
678 auto SrcElementNext = Builder.CreateConstGEP1_32(
679 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
680 // Check whether we've reached the end.
682 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
683 Builder.CreateCondBr(Done, DoneBB, BodyBB);
684 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
685 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
688 EmitBlock(DoneBB, /*IsFinished=*/true);
691 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
692 Address SrcAddr, const VarDecl *DestVD,
693 const VarDecl *SrcVD, const Expr *Copy) {
694 if (OriginalType->isArrayType()) {
695 auto *BO = dyn_cast<BinaryOperator>(Copy);
696 if (BO && BO->getOpcode() == BO_Assign) {
697 // Perform simple memcpy for simple copying.
698 EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
700 // For arrays with complex element types perform element by element
702 EmitOMPAggregateAssign(
703 DestAddr, SrcAddr, OriginalType,
704 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705 // Working with the single array element, so have to remap
706 // destination and source variables to corresponding array
708 CodeGenFunction::OMPPrivateScope Remap(*this);
709 Remap.addPrivate(DestVD, [DestElement]() -> Address {
713 SrcVD, [SrcElement]() -> Address { return SrcElement; });
714 (void)Remap.Privatize();
715 EmitIgnoredExpr(Copy);
719 // Remap pseudo source variable to private copy.
720 CodeGenFunction::OMPPrivateScope Remap(*this);
721 Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
722 Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
723 (void)Remap.Privatize();
724 // Emit copying of the whole variable.
725 EmitIgnoredExpr(Copy);
729 bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
730 OMPPrivateScope &PrivateScope) {
731 if (!HaveInsertPoint())
733 bool FirstprivateIsLastprivate = false;
734 llvm::DenseSet<const VarDecl *> Lastprivates;
735 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
736 for (const auto *D : C->varlists())
738 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
740 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
741 CGCapturedStmtInfo CapturesInfo(cast<CapturedStmt>(*D.getAssociatedStmt()));
742 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
743 auto IRef = C->varlist_begin();
744 auto InitsRef = C->inits().begin();
745 for (auto IInit : C->private_copies()) {
746 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
747 bool ThisFirstprivateIsLastprivate =
748 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
749 auto *CapFD = CapturesInfo.lookup(OrigVD);
750 auto *FD = CapturedStmtInfo->lookup(OrigVD);
751 if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
752 !FD->getType()->isReferenceType()) {
753 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
758 FirstprivateIsLastprivate =
759 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
760 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
761 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
762 auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
764 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
765 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
766 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
767 Address OriginalAddr = EmitLValue(&DRE).getAddress();
768 QualType Type = VD->getType();
769 if (Type->isArrayType()) {
770 // Emit VarDecl with copy init for arrays.
771 // Get the address of the original variable captured in current
773 IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
774 auto Emission = EmitAutoVarAlloca(*VD);
775 auto *Init = VD->getInit();
776 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
777 // Perform simple memcpy.
778 EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
781 EmitOMPAggregateAssign(
782 Emission.getAllocatedAddress(), OriginalAddr, Type,
783 [this, VDInit, Init](Address DestElement,
784 Address SrcElement) {
785 // Clean up any temporaries needed by the initialization.
786 RunCleanupsScope InitScope(*this);
787 // Emit initialization for single element.
788 setAddrOfLocalVar(VDInit, SrcElement);
789 EmitAnyExprToMem(Init, DestElement,
790 Init->getType().getQualifiers(),
791 /*IsInitializer*/ false);
792 LocalDeclMap.erase(VDInit);
795 EmitAutoVarCleanups(Emission);
796 return Emission.getAllocatedAddress();
799 IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
800 // Emit private VarDecl with copy init.
801 // Remap temp VDInit variable to the address of the original
803 // (for proper handling of captured global variables).
804 setAddrOfLocalVar(VDInit, OriginalAddr);
806 LocalDeclMap.erase(VDInit);
807 return GetAddrOfLocalVar(VD);
810 assert(IsRegistered &&
811 "firstprivate var already registered as private");
812 // Silence the warning about unused variable.
819 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
822 void CodeGenFunction::EmitOMPPrivateClause(
823 const OMPExecutableDirective &D,
824 CodeGenFunction::OMPPrivateScope &PrivateScope) {
825 if (!HaveInsertPoint())
827 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
828 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
829 auto IRef = C->varlist_begin();
830 for (auto IInit : C->private_copies()) {
831 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
832 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
833 auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
835 PrivateScope.addPrivate(OrigVD, [&]() -> Address {
836 // Emit private VarDecl with copy init.
838 return GetAddrOfLocalVar(VD);
840 assert(IsRegistered && "private var already registered as private");
841 // Silence the warning about unused variable.
849 bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
850 if (!HaveInsertPoint())
852 // threadprivate_var1 = master_threadprivate_var1;
853 // operator=(threadprivate_var2, master_threadprivate_var2);
855 // __kmpc_barrier(&loc, global_tid);
856 llvm::DenseSet<const VarDecl *> CopiedVars;
857 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
858 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
859 auto IRef = C->varlist_begin();
860 auto ISrcRef = C->source_exprs().begin();
861 auto IDestRef = C->destination_exprs().begin();
862 for (auto *AssignOp : C->assignment_ops()) {
863 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
864 QualType Type = VD->getType();
865 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
866 // Get the address of the master variable. If we are emitting code with
867 // TLS support, the address is passed from the master as field in the
868 // captured declaration.
869 Address MasterAddr = Address::invalid();
870 if (getLangOpts().OpenMPUseTLS &&
871 getContext().getTargetInfo().isTLSSupported()) {
872 assert(CapturedStmtInfo->lookup(VD) &&
873 "Copyin threadprivates should have been captured!");
874 DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
875 VK_LValue, (*IRef)->getExprLoc());
876 MasterAddr = EmitLValue(&DRE).getAddress();
877 LocalDeclMap.erase(VD);
880 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
881 : CGM.GetAddrOfGlobal(VD),
882 getContext().getDeclAlign(VD));
884 // Get the address of the threadprivate variable.
885 Address PrivateAddr = EmitLValue(*IRef).getAddress();
886 if (CopiedVars.size() == 1) {
887 // At first check if current thread is a master thread. If it is, no
888 // need to copy data.
889 CopyBegin = createBasicBlock("copyin.not.master");
890 CopyEnd = createBasicBlock("copyin.not.master.end");
891 Builder.CreateCondBr(
892 Builder.CreateICmpNE(
893 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
894 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
896 EmitBlock(CopyBegin);
898 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
899 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
900 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
908 // Exit out of copying procedure for non-master thread.
909 EmitBlock(CopyEnd, /*IsFinished=*/true);
915 bool CodeGenFunction::EmitOMPLastprivateClauseInit(
916 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
917 if (!HaveInsertPoint())
919 bool HasAtLeastOneLastprivate = false;
920 llvm::DenseSet<const VarDecl *> SIMDLCVs;
921 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
922 auto *LoopDirective = cast<OMPLoopDirective>(&D);
923 for (auto *C : LoopDirective->counters()) {
925 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
928 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
929 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
930 HasAtLeastOneLastprivate = true;
931 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
932 !getLangOpts().OpenMPSimd)
934 auto IRef = C->varlist_begin();
935 auto IDestRef = C->destination_exprs().begin();
936 for (auto *IInit : C->private_copies()) {
937 // Keep the address of the original variable for future update at the end
939 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
940 // Taskloops do not require additional initialization, it is done in
941 // runtime support library.
942 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
943 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
944 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
946 const_cast<VarDecl *>(OrigVD),
947 /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
949 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
950 return EmitLValue(&DRE).getAddress();
952 // Check if the variable is also a firstprivate: in this case IInit is
953 // not generated. Initialization of this variable will happen in codegen
954 // for 'firstprivate' clause.
955 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
956 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
957 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
958 // Emit private VarDecl with copy init.
960 return GetAddrOfLocalVar(VD);
962 assert(IsRegistered &&
963 "lastprivate var already registered as private");
971 return HasAtLeastOneLastprivate;
974 void CodeGenFunction::EmitOMPLastprivateClauseFinal(
975 const OMPExecutableDirective &D, bool NoFinals,
976 llvm::Value *IsLastIterCond) {
977 if (!HaveInsertPoint())
979 // Emit following code:
980 // if (<IsLastIterCond>) {
981 // orig_var1 = private_orig_var1;
983 // orig_varn = private_orig_varn;
985 llvm::BasicBlock *ThenBB = nullptr;
986 llvm::BasicBlock *DoneBB = nullptr;
987 if (IsLastIterCond) {
988 ThenBB = createBasicBlock(".omp.lastprivate.then");
989 DoneBB = createBasicBlock(".omp.lastprivate.done");
990 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
993 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
994 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
995 if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
996 auto IC = LoopDirective->counters().begin();
997 for (auto F : LoopDirective->finals()) {
999 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1001 AlreadyEmittedVars.insert(D);
1003 LoopCountersAndUpdates[D] = F;
1007 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1008 auto IRef = C->varlist_begin();
1009 auto ISrcRef = C->source_exprs().begin();
1010 auto IDestRef = C->destination_exprs().begin();
1011 for (auto *AssignOp : C->assignment_ops()) {
1012 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1013 QualType Type = PrivateVD->getType();
1014 auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1015 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1016 // If lastprivate variable is a loop control variable for loop-based
1017 // directive, update its value before copyin back to original
1019 if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1020 EmitIgnoredExpr(FinalExpr);
1021 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1022 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1023 // Get the address of the original variable.
1024 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1025 // Get the address of the private variable.
1026 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1027 if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1029 Address(Builder.CreateLoad(PrivateAddr),
1030 getNaturalTypeAlignment(RefTy->getPointeeType()));
1031 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1037 if (auto *PostUpdate = C->getPostUpdateExpr())
1038 EmitIgnoredExpr(PostUpdate);
1041 EmitBlock(DoneBB, /*IsFinished=*/true);
1044 void CodeGenFunction::EmitOMPReductionClauseInit(
1045 const OMPExecutableDirective &D,
1046 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1047 if (!HaveInsertPoint())
1049 SmallVector<const Expr *, 4> Shareds;
1050 SmallVector<const Expr *, 4> Privates;
1051 SmallVector<const Expr *, 4> ReductionOps;
1052 SmallVector<const Expr *, 4> LHSs;
1053 SmallVector<const Expr *, 4> RHSs;
1054 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1055 auto IPriv = C->privates().begin();
1056 auto IRed = C->reduction_ops().begin();
1057 auto ILHS = C->lhs_exprs().begin();
1058 auto IRHS = C->rhs_exprs().begin();
1059 for (const auto *Ref : C->varlists()) {
1060 Shareds.emplace_back(Ref);
1061 Privates.emplace_back(*IPriv);
1062 ReductionOps.emplace_back(*IRed);
1063 LHSs.emplace_back(*ILHS);
1064 RHSs.emplace_back(*IRHS);
1065 std::advance(IPriv, 1);
1066 std::advance(IRed, 1);
1067 std::advance(ILHS, 1);
1068 std::advance(IRHS, 1);
1071 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1073 auto ILHS = LHSs.begin();
1074 auto IRHS = RHSs.begin();
1075 auto IPriv = Privates.begin();
1076 for (const auto *IRef : Shareds) {
1077 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1078 // Emit private VarDecl with reduction init.
1079 RedCG.emitSharedLValue(*this, Count);
1080 RedCG.emitAggregateType(*this, Count);
1081 auto Emission = EmitAutoVarAlloca(*PrivateVD);
1082 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1083 RedCG.getSharedLValue(Count),
1084 [&Emission](CodeGenFunction &CGF) {
1085 CGF.EmitAutoVarInit(Emission);
1088 EmitAutoVarCleanups(Emission);
1089 Address BaseAddr = RedCG.adjustPrivateAddress(
1090 *this, Count, Emission.getAllocatedAddress());
1091 bool IsRegistered = PrivateScope.addPrivate(
1092 RedCG.getBaseDecl(Count), [BaseAddr]() -> Address { return BaseAddr; });
1093 assert(IsRegistered && "private var already registered as private");
1094 // Silence the warning about unused variable.
1097 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1098 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1099 QualType Type = PrivateVD->getType();
1100 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1101 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1102 // Store the address of the original variable associated with the LHS
1103 // implicit variable.
1104 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1105 return RedCG.getSharedLValue(Count).getAddress();
1107 PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
1108 return GetAddrOfLocalVar(PrivateVD);
1110 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1111 isa<ArraySubscriptExpr>(IRef)) {
1112 // Store the address of the original variable associated with the LHS
1113 // implicit variable.
1114 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1115 return RedCG.getSharedLValue(Count).getAddress();
1117 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address {
1118 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1119 ConvertTypeForMem(RHSVD->getType()),
1123 QualType Type = PrivateVD->getType();
1124 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1125 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1126 // Store the address of the original variable associated with the LHS
1127 // implicit variable.
1129 OriginalAddr = Builder.CreateElementBitCast(
1130 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1132 PrivateScope.addPrivate(
1133 LHSVD, [OriginalAddr]() -> Address { return OriginalAddr; });
1134 PrivateScope.addPrivate(
1135 RHSVD, [this, PrivateVD, RHSVD, IsArray]() -> Address {
1137 ? Builder.CreateElementBitCast(
1138 GetAddrOfLocalVar(PrivateVD),
1139 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1140 : GetAddrOfLocalVar(PrivateVD);
1150 void CodeGenFunction::EmitOMPReductionClauseFinal(
1151 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1152 if (!HaveInsertPoint())
1154 llvm::SmallVector<const Expr *, 8> Privates;
1155 llvm::SmallVector<const Expr *, 8> LHSExprs;
1156 llvm::SmallVector<const Expr *, 8> RHSExprs;
1157 llvm::SmallVector<const Expr *, 8> ReductionOps;
1158 bool HasAtLeastOneReduction = false;
1159 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1160 HasAtLeastOneReduction = true;
1161 Privates.append(C->privates().begin(), C->privates().end());
1162 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1163 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1164 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1166 if (HasAtLeastOneReduction) {
1167 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1168 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1169 ReductionKind == OMPD_simd;
1170 bool SimpleReduction = ReductionKind == OMPD_simd;
1171 // Emit nowait reduction if nowait clause is present or directive is a
1172 // parallel directive (it always has implicit barrier).
1173 CGM.getOpenMPRuntime().emitReduction(
1174 *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
1175 {WithNowait, SimpleReduction, ReductionKind});
1179 static void emitPostUpdateForReductionClause(
1180 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1181 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1182 if (!CGF.HaveInsertPoint())
1184 llvm::BasicBlock *DoneBB = nullptr;
1185 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1186 if (auto *PostUpdate = C->getPostUpdateExpr()) {
1188 if (auto *Cond = CondGen(CGF)) {
1189 // If the first post-update expression is found, emit conditional
1190 // block if it was requested.
1191 auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1192 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1193 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1194 CGF.EmitBlock(ThenBB);
1197 CGF.EmitIgnoredExpr(PostUpdate);
1201 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1205 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1206 /// parallel function. This is necessary for combined constructs such as
1207 /// 'distribute parallel for'
1208 typedef llvm::function_ref<void(CodeGenFunction &,
1209 const OMPExecutableDirective &,
1210 llvm::SmallVectorImpl<llvm::Value *> &)>
1211 CodeGenBoundParametersTy;
1212 } // anonymous namespace
1214 static void emitCommonOMPParallelDirective(
1215 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1216 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1217 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1218 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1219 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1220 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1221 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1222 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1223 auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1224 /*IgnoreResultAssign*/ true);
1225 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1226 CGF, NumThreads, NumThreadsClause->getLocStart());
1228 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1229 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1230 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1231 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1233 const Expr *IfCond = nullptr;
1234 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1235 if (C->getNameModifier() == OMPD_unknown ||
1236 C->getNameModifier() == OMPD_parallel) {
1237 IfCond = C->getCondition();
1242 OMPParallelScope Scope(CGF, S);
1243 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1244 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1245 // lower and upper bounds with the pragma 'for' chunking mechanism.
1246 // The following lambda takes care of appending the lower and upper bound
1247 // parameters when necessary
1248 CodeGenBoundParameters(CGF, S, CapturedVars);
1249 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1250 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1251 CapturedVars, IfCond);
1254 static void emitEmptyBoundParameters(CodeGenFunction &,
1255 const OMPExecutableDirective &,
1256 llvm::SmallVectorImpl<llvm::Value *> &) {}
1258 void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1259 // Emit parallel region as a standalone region.
1260 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1261 OMPPrivateScope PrivateScope(CGF);
1262 bool Copyins = CGF.EmitOMPCopyinClause(S);
1263 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1265 // Emit implicit barrier to synchronize threads and avoid data races on
1266 // propagation master's thread values of threadprivate variables to local
1267 // instances of that variables of all other implicit threads.
1268 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1269 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1270 /*ForceSimpleCall=*/true);
1272 CGF.EmitOMPPrivateClause(S, PrivateScope);
1273 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1274 (void)PrivateScope.Privatize();
1275 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1276 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1278 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1279 emitEmptyBoundParameters);
1280 emitPostUpdateForReductionClause(
1281 *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1284 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1285 JumpDest LoopExit) {
1286 RunCleanupsScope BodyScope(*this);
1287 // Update counters values on current iteration.
1288 for (auto I : D.updates()) {
1291 // Update the linear variables.
1292 // In distribute directives only loop counters may be marked as linear, no
1293 // need to generate the code for them.
1294 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1295 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1296 for (auto *U : C->updates())
1301 // On a continue in the body, jump to the end.
1302 auto Continue = getJumpDestInCurrentScope("omp.body.continue");
1303 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1305 EmitStmt(D.getBody());
1306 // The end (updates/cleanups).
1307 EmitBlock(Continue.getBlock());
1308 BreakContinueStack.pop_back();
1311 void CodeGenFunction::EmitOMPInnerLoop(
1312 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1313 const Expr *IncExpr,
1314 const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
1315 const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
1316 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1318 // Start the loop with a block that tests the condition.
1319 auto CondBlock = createBasicBlock("omp.inner.for.cond");
1320 EmitBlock(CondBlock);
1321 const SourceRange &R = S.getSourceRange();
1322 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1323 SourceLocToDebugLoc(R.getEnd()));
1325 // If there are any cleanups between here and the loop-exit scope,
1326 // create a block to stage a loop exit along.
1327 auto ExitBlock = LoopExit.getBlock();
1328 if (RequiresCleanup)
1329 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1331 auto LoopBody = createBasicBlock("omp.inner.for.body");
1334 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1335 if (ExitBlock != LoopExit.getBlock()) {
1336 EmitBlock(ExitBlock);
1337 EmitBranchThroughCleanup(LoopExit);
1340 EmitBlock(LoopBody);
1341 incrementProfileCounter(&S);
1343 // Create a block for the increment.
1344 auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1345 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1349 // Emit "IV = IV + 1" and a back-edge to the condition block.
1350 EmitBlock(Continue.getBlock());
1351 EmitIgnoredExpr(IncExpr);
1353 BreakContinueStack.pop_back();
1354 EmitBranch(CondBlock);
1356 // Emit the fall-through block.
1357 EmitBlock(LoopExit.getBlock());
1360 bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
1361 if (!HaveInsertPoint())
1363 // Emit inits for the linear variables.
1364 bool HasLinears = false;
1365 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1366 for (auto *Init : C->inits()) {
1368 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1369 if (auto *Ref = dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1370 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1371 auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1372 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1373 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1374 VD->getInit()->getType(), VK_LValue,
1375 VD->getInit()->getExprLoc());
1376 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1378 /*capturedByInit=*/false);
1379 EmitAutoVarCleanups(Emission);
1383 // Emit the linear steps for the linear clauses.
1384 // If a step is not constant, it is pre-calculated before the loop.
1385 if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1386 if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1387 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1388 // Emit calculation of the linear step.
1389 EmitIgnoredExpr(CS);
1395 void CodeGenFunction::EmitOMPLinearClauseFinal(
1396 const OMPLoopDirective &D,
1397 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1398 if (!HaveInsertPoint())
1400 llvm::BasicBlock *DoneBB = nullptr;
1401 // Emit the final values of the linear variables.
1402 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1403 auto IC = C->varlist_begin();
1404 for (auto *F : C->finals()) {
1406 if (auto *Cond = CondGen(*this)) {
1407 // If the first post-update expression is found, emit conditional
1408 // block if it was requested.
1409 auto *ThenBB = createBasicBlock(".omp.linear.pu");
1410 DoneBB = createBasicBlock(".omp.linear.pu.done");
1411 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1415 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1416 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1417 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1418 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1419 Address OrigAddr = EmitLValue(&DRE).getAddress();
1420 CodeGenFunction::OMPPrivateScope VarScope(*this);
1421 VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; });
1422 (void)VarScope.Privatize();
1426 if (auto *PostUpdate = C->getPostUpdateExpr())
1427 EmitIgnoredExpr(PostUpdate);
1430 EmitBlock(DoneBB, /*IsFinished=*/true);
1433 static void emitAlignedClause(CodeGenFunction &CGF,
1434 const OMPExecutableDirective &D) {
1435 if (!CGF.HaveInsertPoint())
1437 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1438 unsigned ClauseAlignment = 0;
1439 if (auto AlignmentExpr = Clause->getAlignment()) {
1441 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1442 ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1444 for (auto E : Clause->varlists()) {
1445 unsigned Alignment = ClauseAlignment;
1446 if (Alignment == 0) {
1447 // OpenMP [2.8.1, Description]
1448 // If no optional parameter is specified, implementation-defined default
1449 // alignments for SIMD instructions on the target platforms are assumed.
1452 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1453 E->getType()->getPointeeType()))
1456 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1457 "alignment is not power of 2");
1458 if (Alignment != 0) {
1459 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1460 CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1466 void CodeGenFunction::EmitOMPPrivateLoopCounters(
1467 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1468 if (!HaveInsertPoint())
1470 auto I = S.private_counters().begin();
1471 for (auto *E : S.counters()) {
1472 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1473 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1474 (void)LoopScope.addPrivate(VD, [&]() -> Address {
1475 // Emit var without initialization.
1476 if (!LocalDeclMap.count(PrivateVD)) {
1477 auto VarEmission = EmitAutoVarAlloca(*PrivateVD);
1478 EmitAutoVarCleanups(VarEmission);
1480 DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1481 /*RefersToEnclosingVariableOrCapture=*/false,
1482 (*I)->getType(), VK_LValue, (*I)->getExprLoc());
1483 return EmitLValue(&DRE).getAddress();
1485 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1486 VD->hasGlobalStorage()) {
1487 (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
1488 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1489 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1490 E->getType(), VK_LValue, E->getExprLoc());
1491 return EmitLValue(&DRE).getAddress();
1498 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1499 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1500 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1501 if (!CGF.HaveInsertPoint())
1504 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1505 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1506 (void)PreCondScope.Privatize();
1507 // Get initial values of real counters.
1508 for (auto I : S.inits()) {
1509 CGF.EmitIgnoredExpr(I);
1512 // Check that loop is executed at least one time.
1513 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1516 void CodeGenFunction::EmitOMPLinearClause(
1517 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1518 if (!HaveInsertPoint())
1520 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1521 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1522 auto *LoopDirective = cast<OMPLoopDirective>(&D);
1523 for (auto *C : LoopDirective->counters()) {
1525 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1528 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1529 auto CurPrivate = C->privates().begin();
1530 for (auto *E : C->varlists()) {
1531 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1533 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1534 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1535 bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
1536 // Emit private VarDecl with copy init.
1537 EmitVarDecl(*PrivateVD);
1538 return GetAddrOfLocalVar(PrivateVD);
1540 assert(IsRegistered && "linear var already registered as private");
1541 // Silence the warning about unused variable.
1544 EmitVarDecl(*PrivateVD);
1550 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1551 const OMPExecutableDirective &D,
1553 if (!CGF.HaveInsertPoint())
1555 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1556 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1557 /*ignoreResult=*/true);
1558 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1559 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1560 // In presence of finite 'safelen', it may be unsafe to mark all
1561 // the memory instructions parallel, because loop-carried
1562 // dependences of 'safelen' iterations are possible.
1564 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1565 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1566 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1567 /*ignoreResult=*/true);
1568 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1569 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1570 // In presence of finite 'safelen', it may be unsafe to mark all
1571 // the memory instructions parallel, because loop-carried
1572 // dependences of 'safelen' iterations are possible.
1573 CGF.LoopStack.setParallel(false);
1577 void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1579 // Walk clauses and process safelen/lastprivate.
1580 LoopStack.setParallel(!IsMonotonic);
1581 LoopStack.setVectorizeEnable(true);
1582 emitSimdlenSafelenClause(*this, D, IsMonotonic);
1585 void CodeGenFunction::EmitOMPSimdFinal(
1586 const OMPLoopDirective &D,
1587 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1588 if (!HaveInsertPoint())
1590 llvm::BasicBlock *DoneBB = nullptr;
1591 auto IC = D.counters().begin();
1592 auto IPC = D.private_counters().begin();
1593 for (auto F : D.finals()) {
1594 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1595 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1596 auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1597 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1598 OrigVD->hasGlobalStorage() || CED) {
1600 if (auto *Cond = CondGen(*this)) {
1601 // If the first post-update expression is found, emit conditional
1602 // block if it was requested.
1603 auto *ThenBB = createBasicBlock(".omp.final.then");
1604 DoneBB = createBasicBlock(".omp.final.done");
1605 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1609 Address OrigAddr = Address::invalid();
1611 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1613 DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1614 /*RefersToEnclosingVariableOrCapture=*/false,
1615 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1616 OrigAddr = EmitLValue(&DRE).getAddress();
1618 OMPPrivateScope VarScope(*this);
1619 VarScope.addPrivate(OrigVD,
1620 [OrigAddr]() -> Address { return OrigAddr; });
1621 (void)VarScope.Privatize();
1628 EmitBlock(DoneBB, /*IsFinished=*/true);
1631 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1632 const OMPLoopDirective &S,
1633 CodeGenFunction::JumpDest LoopExit) {
1634 CGF.EmitOMPLoopBody(S, LoopExit);
1635 CGF.EmitStopPoint(&S);
1638 /// Emit a helper variable and return corresponding lvalue.
1639 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1640 const DeclRefExpr *Helper) {
1641 auto VDecl = cast<VarDecl>(Helper->getDecl());
1642 CGF.EmitVarDecl(*VDecl);
1643 return CGF.EmitLValue(Helper);
1646 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1647 PrePostActionTy &Action) {
1649 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
1650 "Expected simd directive");
1651 OMPLoopScope PreInitScope(CGF, S);
1653 // for (IV in 0..LastIteration) BODY;
1654 // <Final counter/linear vars updates>;
1657 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
1658 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
1659 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
1660 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1661 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1664 // Emit: if (PreCond) - begin.
1665 // If the condition constant folds and can be elided, avoid emitting the
1668 llvm::BasicBlock *ContBlock = nullptr;
1669 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1673 auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1674 ContBlock = CGF.createBasicBlock("simd.if.end");
1675 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1676 CGF.getProfileCount(&S));
1677 CGF.EmitBlock(ThenBlock);
1678 CGF.incrementProfileCounter(&S);
1681 // Emit the loop iteration variable.
1682 const Expr *IVExpr = S.getIterationVariable();
1683 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1684 CGF.EmitVarDecl(*IVDecl);
1685 CGF.EmitIgnoredExpr(S.getInit());
1687 // Emit the iterations count variable.
1688 // If it is not a variable, Sema decided to calculate iterations count on
1689 // each iteration (e.g., it is foldable into a constant).
1690 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1691 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1692 // Emit calculation of the iterations count.
1693 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1696 CGF.EmitOMPSimdInit(S);
1698 emitAlignedClause(CGF, S);
1699 (void)CGF.EmitOMPLinearClauseInit(S);
1701 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1702 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1703 CGF.EmitOMPLinearClause(S, LoopScope);
1704 CGF.EmitOMPPrivateClause(S, LoopScope);
1705 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1706 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1707 (void)LoopScope.Privatize();
1708 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1710 [&S](CodeGenFunction &CGF) {
1711 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
1712 CGF.EmitStopPoint(&S);
1714 [](CodeGenFunction &) {});
1715 CGF.EmitOMPSimdFinal(
1716 S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1717 // Emit final copy of the lastprivate variables at the end of loops.
1718 if (HasLastprivateClause)
1719 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1720 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1721 emitPostUpdateForReductionClause(
1722 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1724 CGF.EmitOMPLinearClauseFinal(
1725 S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1726 // Emit: if (PreCond) - end.
1728 CGF.EmitBranch(ContBlock);
1729 CGF.EmitBlock(ContBlock, true);
1733 void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1734 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1735 emitOMPSimdRegion(CGF, S, Action);
1737 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1738 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1741 void CodeGenFunction::EmitOMPOuterLoop(
1742 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1743 CodeGenFunction::OMPPrivateScope &LoopScope,
1744 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1745 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1746 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1747 auto &RT = CGM.getOpenMPRuntime();
1749 const Expr *IVExpr = S.getIterationVariable();
1750 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1751 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1753 auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1755 // Start the loop with a block that tests the condition.
1756 auto CondBlock = createBasicBlock("omp.dispatch.cond");
1757 EmitBlock(CondBlock);
1758 const SourceRange &R = S.getSourceRange();
1759 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1760 SourceLocToDebugLoc(R.getEnd()));
1762 llvm::Value *BoolCondVal = nullptr;
1763 if (!DynamicOrOrdered) {
1764 // UB = min(UB, GlobalUB) or
1765 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1766 // 'distribute parallel for')
1767 EmitIgnoredExpr(LoopArgs.EUB);
1769 EmitIgnoredExpr(LoopArgs.Init);
1771 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1774 RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1775 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1778 // If there are any cleanups between here and the loop-exit scope,
1779 // create a block to stage a loop exit along.
1780 auto ExitBlock = LoopExit.getBlock();
1781 if (LoopScope.requiresCleanups())
1782 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1784 auto LoopBody = createBasicBlock("omp.dispatch.body");
1785 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1786 if (ExitBlock != LoopExit.getBlock()) {
1787 EmitBlock(ExitBlock);
1788 EmitBranchThroughCleanup(LoopExit);
1790 EmitBlock(LoopBody);
1792 // Emit "IV = LB" (in case of static schedule, we have already calculated new
1793 // LB for loop condition and emitted it above).
1794 if (DynamicOrOrdered)
1795 EmitIgnoredExpr(LoopArgs.Init);
1797 // Create a block for the increment.
1798 auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1799 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1801 // Generate !llvm.loop.parallel metadata for loads and stores for loops
1802 // with dynamic/guided scheduling and without ordered clause.
1803 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
1804 LoopStack.setParallel(!IsMonotonic);
1806 EmitOMPSimdInit(S, IsMonotonic);
1808 SourceLocation Loc = S.getLocStart();
1810 // when 'distribute' is not combined with a 'for':
1811 // while (idx <= UB) { BODY; ++idx; }
1812 // when 'distribute' is combined with a 'for'
1813 // (e.g. 'distribute parallel for')
1814 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1816 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1817 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1818 CodeGenLoop(CGF, S, LoopExit);
1820 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1821 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1824 EmitBlock(Continue.getBlock());
1825 BreakContinueStack.pop_back();
1826 if (!DynamicOrOrdered) {
1827 // Emit "LB = LB + Stride", "UB = UB + Stride".
1828 EmitIgnoredExpr(LoopArgs.NextLB);
1829 EmitIgnoredExpr(LoopArgs.NextUB);
1832 EmitBranch(CondBlock);
1834 // Emit the fall-through block.
1835 EmitBlock(LoopExit.getBlock());
1837 // Tell the runtime we are done.
1838 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1839 if (!DynamicOrOrdered)
1840 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
1841 S.getDirectiveKind());
1843 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1846 void CodeGenFunction::EmitOMPForOuterLoop(
1847 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1848 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1849 const OMPLoopArguments &LoopArgs,
1850 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1851 auto &RT = CGM.getOpenMPRuntime();
1853 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1854 const bool DynamicOrOrdered =
1855 Ordered || RT.isDynamic(ScheduleKind.Schedule);
1858 !RT.isStaticNonchunked(ScheduleKind.Schedule,
1859 LoopArgs.Chunk != nullptr)) &&
1860 "static non-chunked schedule does not need outer loop");
1864 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1865 // When schedule(dynamic,chunk_size) is specified, the iterations are
1866 // distributed to threads in the team in chunks as the threads request them.
1867 // Each thread executes a chunk of iterations, then requests another chunk,
1868 // until no chunks remain to be distributed. Each chunk contains chunk_size
1869 // iterations, except for the last chunk to be distributed, which may have
1870 // fewer iterations. When no chunk_size is specified, it defaults to 1.
1872 // When schedule(guided,chunk_size) is specified, the iterations are assigned
1873 // to threads in the team in chunks as the executing threads request them.
1874 // Each thread executes a chunk of iterations, then requests another chunk,
1875 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1876 // each chunk is proportional to the number of unassigned iterations divided
1877 // by the number of threads in the team, decreasing to 1. For a chunk_size
1878 // with value k (greater than 1), the size of each chunk is determined in the
1879 // same way, with the restriction that the chunks do not contain fewer than k
1880 // iterations (except for the last chunk to be assigned, which may have fewer
1881 // than k iterations).
1883 // When schedule(auto) is specified, the decision regarding scheduling is
1884 // delegated to the compiler and/or runtime system. The programmer gives the
1885 // implementation the freedom to choose any possible mapping of iterations to
1886 // threads in the team.
1888 // When schedule(runtime) is specified, the decision regarding scheduling is
1889 // deferred until run time, and the schedule and chunk size are taken from the
1890 // run-sched-var ICV. If the ICV is set to auto, the schedule is
1891 // implementation defined
1893 // while(__kmpc_dispatch_next(&LB, &UB)) {
1895 // while (idx <= UB) { BODY; ++idx;
1896 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1900 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1901 // When schedule(static, chunk_size) is specified, iterations are divided into
1902 // chunks of size chunk_size, and the chunks are assigned to the threads in
1903 // the team in a round-robin fashion in the order of the thread number.
1905 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1906 // while (idx <= UB) { BODY; ++idx; } // inner loop
1912 const Expr *IVExpr = S.getIterationVariable();
1913 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1914 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1916 if (DynamicOrOrdered) {
1917 auto DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1918 llvm::Value *LBVal = DispatchBounds.first;
1919 llvm::Value *UBVal = DispatchBounds.second;
1920 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1922 RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1923 IVSigned, Ordered, DipatchRTInputValues);
1925 CGOpenMPRuntime::StaticRTInput StaticInit(
1926 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1927 LoopArgs.ST, LoopArgs.Chunk);
1928 RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
1929 ScheduleKind, StaticInit);
1932 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1933 const unsigned IVSize,
1934 const bool IVSigned) {
1936 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1941 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1942 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1943 OuterLoopArgs.IncExpr = S.getInc();
1944 OuterLoopArgs.Init = S.getInit();
1945 OuterLoopArgs.Cond = S.getCond();
1946 OuterLoopArgs.NextLB = S.getNextLowerBound();
1947 OuterLoopArgs.NextUB = S.getNextUpperBound();
1948 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1949 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1952 static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
1953 const unsigned IVSize, const bool IVSigned) {}
1955 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1956 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1957 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1958 const CodeGenLoopTy &CodeGenLoopContent) {
1960 auto &RT = CGM.getOpenMPRuntime();
1963 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1967 const Expr *IVExpr = S.getIterationVariable();
1968 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1969 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1971 CGOpenMPRuntime::StaticRTInput StaticInit(
1972 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
1973 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
1974 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit);
1976 // for combined 'distribute' and 'for' the increment expression of distribute
1977 // is store in DistInc. For 'distribute' alone, it is in Inc.
1979 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
1980 IncExpr = S.getDistInc();
1982 IncExpr = S.getInc();
1984 // this routine is shared by 'omp distribute parallel for' and
1985 // 'omp distribute': select the right EUB expression depending on the
1987 OMPLoopArguments OuterLoopArgs;
1988 OuterLoopArgs.LB = LoopArgs.LB;
1989 OuterLoopArgs.UB = LoopArgs.UB;
1990 OuterLoopArgs.ST = LoopArgs.ST;
1991 OuterLoopArgs.IL = LoopArgs.IL;
1992 OuterLoopArgs.Chunk = LoopArgs.Chunk;
1993 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1994 ? S.getCombinedEnsureUpperBound()
1995 : S.getEnsureUpperBound();
1996 OuterLoopArgs.IncExpr = IncExpr;
1997 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1998 ? S.getCombinedInit()
2000 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2001 ? S.getCombinedCond()
2003 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2004 ? S.getCombinedNextLowerBound()
2005 : S.getNextLowerBound();
2006 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2007 ? S.getCombinedNextUpperBound()
2008 : S.getNextUpperBound();
2010 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2011 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2015 static std::pair<LValue, LValue>
2016 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
2017 const OMPExecutableDirective &S) {
2018 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2020 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2022 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2024 // When composing 'distribute' with 'for' (e.g. as in 'distribute
2025 // parallel for') we need to use the 'distribute'
2026 // chunk lower and upper bounds rather than the whole loop iteration
2027 // space. These are parameters to the outlined function for 'parallel'
2028 // and we copy the bounds of the previous schedule into the
2029 // the current ones.
2030 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2031 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2032 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(PrevLB, SourceLocation());
2033 PrevLBVal = CGF.EmitScalarConversion(
2034 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2035 LS.getIterationVariable()->getType(), SourceLocation());
2036 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(PrevUB, SourceLocation());
2037 PrevUBVal = CGF.EmitScalarConversion(
2038 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2039 LS.getIterationVariable()->getType(), SourceLocation());
2041 CGF.EmitStoreOfScalar(PrevLBVal, LB);
2042 CGF.EmitStoreOfScalar(PrevUBVal, UB);
2047 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2048 /// we need to use the LB and UB expressions generated by the worksharing
2049 /// code generation support, whereas in non combined situations we would
2050 /// just emit 0 and the LastIteration expression
2051 /// This function is necessary due to the difference of the LB and UB
2052 /// types for the RT emission routines for 'for_static_init' and
2053 /// 'for_dispatch_init'
2054 static std::pair<llvm::Value *, llvm::Value *>
2055 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
2056 const OMPExecutableDirective &S,
2057 Address LB, Address UB) {
2058 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2059 const Expr *IVExpr = LS.getIterationVariable();
2060 // when implementing a dynamic schedule for a 'for' combined with a
2061 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2062 // is not normalized as each team only executes its own assigned
2064 QualType IteratorTy = IVExpr->getType();
2065 llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy,
2067 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy,
2069 return {LBVal, UBVal};
2072 static void emitDistributeParallelForDistributeInnerBoundParams(
2073 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2074 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2075 const auto &Dir = cast<OMPLoopDirective>(S);
2077 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2078 auto LBCast = CGF.Builder.CreateIntCast(
2079 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2080 CapturedVars.push_back(LBCast);
2082 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2084 auto UBCast = CGF.Builder.CreateIntCast(
2085 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2086 CapturedVars.push_back(UBCast);
2090 emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2091 const OMPLoopDirective &S,
2092 CodeGenFunction::JumpDest LoopExit) {
2093 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2094 PrePostActionTy &) {
2095 bool HasCancel = false;
2096 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2097 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2098 HasCancel = D->hasCancel();
2099 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2100 HasCancel = D->hasCancel();
2101 else if (const auto *D =
2102 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2103 HasCancel = D->hasCancel();
2105 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2107 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2108 emitDistributeParallelForInnerBounds,
2109 emitDistributeParallelForDispatchBounds);
2112 emitCommonOMPParallelDirective(
2114 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2115 CGInlinedWorksharingLoop,
2116 emitDistributeParallelForDistributeInnerBoundParams);
2119 void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2120 const OMPDistributeParallelForDirective &S) {
2121 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2122 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2125 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2126 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2129 void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2130 const OMPDistributeParallelForSimdDirective &S) {
2131 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2132 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2135 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2136 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2139 void CodeGenFunction::EmitOMPDistributeSimdDirective(
2140 const OMPDistributeSimdDirective &S) {
2141 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2142 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2144 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2145 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2148 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2149 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2150 // Emit SPMD target parallel for region as a standalone region.
2151 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2152 emitOMPSimdRegion(CGF, S, Action);
2155 llvm::Constant *Addr;
2156 // Emit target region as a standalone region.
2157 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2158 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2159 assert(Fn && Addr && "Target device function emission failed.");
2162 void CodeGenFunction::EmitOMPTargetSimdDirective(
2163 const OMPTargetSimdDirective &S) {
2164 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2165 emitOMPSimdRegion(CGF, S, Action);
2167 emitCommonOMPTargetDirective(*this, S, CodeGen);
2170 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
2171 const OMPTargetTeamsDistributeParallelForDirective &S) {
2172 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2173 CGM.getOpenMPRuntime().emitInlinedDirective(
2174 *this, OMPD_target_teams_distribute_parallel_for,
2175 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2177 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2181 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
2182 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
2183 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2184 CGM.getOpenMPRuntime().emitInlinedDirective(
2185 *this, OMPD_target_teams_distribute_parallel_for_simd,
2186 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2188 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2193 struct ScheduleKindModifiersTy {
2194 OpenMPScheduleClauseKind Kind;
2195 OpenMPScheduleClauseModifier M1;
2196 OpenMPScheduleClauseModifier M2;
2197 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2198 OpenMPScheduleClauseModifier M1,
2199 OpenMPScheduleClauseModifier M2)
2200 : Kind(Kind), M1(M1), M2(M2) {}
2204 bool CodeGenFunction::EmitOMPWorksharingLoop(
2205 const OMPLoopDirective &S, Expr *EUB,
2206 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2207 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2208 // Emit the loop iteration variable.
2209 auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2210 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2211 EmitVarDecl(*IVDecl);
2213 // Emit the iterations count variable.
2214 // If it is not a variable, Sema decided to calculate iterations count on each
2215 // iteration (e.g., it is foldable into a constant).
2216 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2217 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2218 // Emit calculation of the iterations count.
2219 EmitIgnoredExpr(S.getCalcLastIteration());
2222 auto &RT = CGM.getOpenMPRuntime();
2224 bool HasLastprivateClause;
2225 // Check pre-condition.
2227 OMPLoopScope PreInitScope(*this, S);
2228 // Skip the entire loop if we don't meet the precondition.
2229 // If the condition constant folds and can be elided, avoid emitting the
2232 llvm::BasicBlock *ContBlock = nullptr;
2233 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2237 auto *ThenBlock = createBasicBlock("omp.precond.then");
2238 ContBlock = createBasicBlock("omp.precond.end");
2239 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2240 getProfileCount(&S));
2241 EmitBlock(ThenBlock);
2242 incrementProfileCounter(&S);
2245 bool Ordered = false;
2246 if (auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2247 if (OrderedClause->getNumForLoops())
2248 RT.emitDoacrossInit(*this, S);
2253 llvm::DenseSet<const Expr *> EmittedFinals;
2254 emitAlignedClause(*this, S);
2255 bool HasLinears = EmitOMPLinearClauseInit(S);
2256 // Emit helper vars inits.
2258 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2259 LValue LB = Bounds.first;
2260 LValue UB = Bounds.second;
2262 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2264 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2266 // Emit 'then' code.
2268 OMPPrivateScope LoopScope(*this);
2269 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2270 // Emit implicit barrier to synchronize threads and avoid data races on
2271 // initialization of firstprivate variables and post-update of
2272 // lastprivate variables.
2273 CGM.getOpenMPRuntime().emitBarrierCall(
2274 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2275 /*ForceSimpleCall=*/true);
2277 EmitOMPPrivateClause(S, LoopScope);
2278 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2279 EmitOMPReductionClauseInit(S, LoopScope);
2280 EmitOMPPrivateLoopCounters(S, LoopScope);
2281 EmitOMPLinearClause(S, LoopScope);
2282 (void)LoopScope.Privatize();
2284 // Detect the loop schedule kind and chunk.
2285 llvm::Value *Chunk = nullptr;
2286 OpenMPScheduleTy ScheduleKind;
2287 if (auto *C = S.getSingleClause<OMPScheduleClause>()) {
2288 ScheduleKind.Schedule = C->getScheduleKind();
2289 ScheduleKind.M1 = C->getFirstScheduleModifier();
2290 ScheduleKind.M2 = C->getSecondScheduleModifier();
2291 if (const auto *Ch = C->getChunkSize()) {
2292 Chunk = EmitScalarExpr(Ch);
2293 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2294 S.getIterationVariable()->getType(),
2298 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2299 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2300 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2301 // If the static schedule kind is specified or if the ordered clause is
2302 // specified, and if no monotonic modifier is specified, the effect will
2303 // be as if the monotonic modifier was specified.
2304 if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2305 /* Chunked */ Chunk != nullptr) &&
2307 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2308 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2309 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2310 // When no chunk_size is specified, the iteration space is divided into
2311 // chunks that are approximately equal in size, and at most one chunk is
2312 // distributed to each thread. Note that the size of the chunks is
2313 // unspecified in this case.
2314 CGOpenMPRuntime::StaticRTInput StaticInit(
2315 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2316 UB.getAddress(), ST.getAddress());
2317 RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
2318 ScheduleKind, StaticInit);
2320 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2321 // UB = min(UB, GlobalUB);
2322 EmitIgnoredExpr(S.getEnsureUpperBound());
2324 EmitIgnoredExpr(S.getInit());
2325 // while (idx <= UB) { BODY; ++idx; }
2326 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2328 [&S, LoopExit](CodeGenFunction &CGF) {
2329 CGF.EmitOMPLoopBody(S, LoopExit);
2330 CGF.EmitStopPoint(&S);
2332 [](CodeGenFunction &) {});
2333 EmitBlock(LoopExit.getBlock());
2334 // Tell the runtime we are done.
2335 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2336 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2337 S.getDirectiveKind());
2339 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2341 const bool IsMonotonic =
2342 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2343 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2344 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2345 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2346 // Emit the outer loop, which requests its work chunk [LB..UB] from
2347 // runtime and runs the inner loop to process it.
2348 const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2349 ST.getAddress(), IL.getAddress(),
2351 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2352 LoopArguments, CGDispatchBounds);
2354 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2356 [&](CodeGenFunction &CGF) -> llvm::Value * {
2357 return CGF.Builder.CreateIsNotNull(
2358 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2361 EmitOMPReductionClauseFinal(
2362 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2363 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2364 : /*Parallel only*/ OMPD_parallel);
2365 // Emit post-update of the reduction variables if IsLastIter != 0.
2366 emitPostUpdateForReductionClause(
2367 *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2368 return CGF.Builder.CreateIsNotNull(
2369 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2371 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2372 if (HasLastprivateClause)
2373 EmitOMPLastprivateClauseFinal(
2374 S, isOpenMPSimdDirective(S.getDirectiveKind()),
2375 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
2377 EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2378 return CGF.Builder.CreateIsNotNull(
2379 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2381 // We're now done with the loop, so jump to the continuation block.
2383 EmitBranch(ContBlock);
2384 EmitBlock(ContBlock, true);
2387 return HasLastprivateClause;
2390 /// The following two functions generate expressions for the loop lower
2391 /// and upper bounds in case of static and dynamic (dispatch) schedule
2392 /// of the associated 'for' or 'distribute' loop.
2393 static std::pair<LValue, LValue>
2394 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2395 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2397 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2399 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2403 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2404 /// consider the lower and upper bound expressions generated by the
2405 /// worksharing loop support, but we use 0 and the iteration space size as
2407 static std::pair<llvm::Value *, llvm::Value *>
2408 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2409 Address LB, Address UB) {
2410 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2411 const Expr *IVExpr = LS.getIterationVariable();
2412 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2413 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2414 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2415 return {LBVal, UBVal};
2418 void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
2419 bool HasLastprivates = false;
2420 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2421 PrePostActionTy &) {
2422 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2423 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2425 emitDispatchForLoopBounds);
2428 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2429 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2433 // Emit an implicit barrier at the end.
2434 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2435 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2439 void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
2440 bool HasLastprivates = false;
2441 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2442 PrePostActionTy &) {
2443 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2445 emitDispatchForLoopBounds);
2448 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2449 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2452 // Emit an implicit barrier at the end.
2453 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2454 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2458 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2460 llvm::Value *Init = nullptr) {
2461 auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2463 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2467 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2468 auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
2469 auto *CS = dyn_cast<CompoundStmt>(Stmt);
2470 bool HasLastprivates = false;
2471 auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF,
2472 PrePostActionTy &) {
2473 auto &C = CGF.CGM.getContext();
2474 auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2475 // Emit helper vars inits.
2476 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2477 CGF.Builder.getInt32(0));
2478 auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2479 : CGF.Builder.getInt32(0);
2481 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2482 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2483 CGF.Builder.getInt32(1));
2484 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2485 CGF.Builder.getInt32(0));
2487 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2488 OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2489 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2490 OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2491 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2492 // Generate condition for loop.
2493 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2494 OK_Ordinary, S.getLocStart(), FPOptions());
2495 // Increment for loop counter.
2496 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2498 auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
2499 // Iterate through all sections and emit a switch construct:
2502 // <SectionStmt[0]>;
2505 // case <NumSection> - 1:
2506 // <SectionStmt[<NumSection> - 1]>;
2509 // .omp.sections.exit:
2510 auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2511 auto *SwitchStmt = CGF.Builder.CreateSwitch(
2512 CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2513 CS == nullptr ? 1 : CS->size());
2515 unsigned CaseNumber = 0;
2516 for (auto *SubStmt : CS->children()) {
2517 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2518 CGF.EmitBlock(CaseBB);
2519 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2520 CGF.EmitStmt(SubStmt);
2521 CGF.EmitBranch(ExitBB);
2525 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2526 CGF.EmitBlock(CaseBB);
2527 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2529 CGF.EmitBranch(ExitBB);
2531 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2534 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2535 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2536 // Emit implicit barrier to synchronize threads and avoid data races on
2537 // initialization of firstprivate variables and post-update of lastprivate
2539 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2540 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2541 /*ForceSimpleCall=*/true);
2543 CGF.EmitOMPPrivateClause(S, LoopScope);
2544 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2545 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2546 (void)LoopScope.Privatize();
2548 // Emit static non-chunked loop.
2549 OpenMPScheduleTy ScheduleKind;
2550 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2551 CGOpenMPRuntime::StaticRTInput StaticInit(
2552 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2553 LB.getAddress(), UB.getAddress(), ST.getAddress());
2554 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2555 CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2556 // UB = min(UB, GlobalUB);
2557 auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2558 auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2559 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2560 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2562 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2563 // while (idx <= UB) { BODY; ++idx; }
2564 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2565 [](CodeGenFunction &) {});
2566 // Tell the runtime we are done.
2567 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2568 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2569 S.getDirectiveKind());
2571 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2572 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2573 // Emit post-update of the reduction variables if IsLastIter != 0.
2574 emitPostUpdateForReductionClause(
2575 CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2576 return CGF.Builder.CreateIsNotNull(
2577 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2580 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2581 if (HasLastprivates)
2582 CGF.EmitOMPLastprivateClauseFinal(
2583 S, /*NoFinals=*/false,
2584 CGF.Builder.CreateIsNotNull(
2585 CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2588 bool HasCancel = false;
2589 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2590 HasCancel = OSD->hasCancel();
2591 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2592 HasCancel = OPSD->hasCancel();
2593 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2594 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2596 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2597 // clause. Otherwise the barrier will be generated by the codegen for the
2599 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2600 // Emit implicit barrier to synchronize threads and avoid data races on
2601 // initialization of firstprivate variables.
2602 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2607 void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
2609 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2612 // Emit an implicit barrier at the end.
2613 if (!S.getSingleClause<OMPNowaitClause>()) {
2614 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2619 void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
2620 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2621 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2623 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2624 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2628 void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
2629 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2630 llvm::SmallVector<const Expr *, 8> DestExprs;
2631 llvm::SmallVector<const Expr *, 8> SrcExprs;
2632 llvm::SmallVector<const Expr *, 8> AssignmentOps;
2633 // Check if there are any 'copyprivate' clauses associated with this
2634 // 'single' construct.
2635 // Build a list of copyprivate variables along with helper expressions
2636 // (<source>, <destination>, <destination>=<source> expressions)
2637 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2638 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2639 DestExprs.append(C->destination_exprs().begin(),
2640 C->destination_exprs().end());
2641 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2642 AssignmentOps.append(C->assignment_ops().begin(),
2643 C->assignment_ops().end());
2645 // Emit code for 'single' region along with 'copyprivate' clauses
2646 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2648 OMPPrivateScope SingleScope(CGF);
2649 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2650 CGF.EmitOMPPrivateClause(S, SingleScope);
2651 (void)SingleScope.Privatize();
2652 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2655 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2656 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2657 CopyprivateVars, DestExprs,
2658 SrcExprs, AssignmentOps);
2660 // Emit an implicit barrier at the end (to avoid data race on firstprivate
2661 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2662 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2663 CGM.getOpenMPRuntime().emitBarrierCall(
2664 *this, S.getLocStart(),
2665 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2669 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
2670 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2672 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2674 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2675 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
2678 void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
2679 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2681 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2683 Expr *Hint = nullptr;
2684 if (auto *HintClause = S.getSingleClause<OMPHintClause>())
2685 Hint = HintClause->getHint();
2686 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2687 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2688 S.getDirectiveName().getAsString(),
2689 CodeGen, S.getLocStart(), Hint);
2692 void CodeGenFunction::EmitOMPParallelForDirective(
2693 const OMPParallelForDirective &S) {
2694 // Emit directive as a combined directive that consists of two implicit
2695 // directives: 'parallel' with 'for' directive.
2696 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2697 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2698 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2699 emitDispatchForLoopBounds);
2701 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2702 emitEmptyBoundParameters);
2705 void CodeGenFunction::EmitOMPParallelForSimdDirective(
2706 const OMPParallelForSimdDirective &S) {
2707 // Emit directive as a combined directive that consists of two implicit
2708 // directives: 'parallel' with 'for' directive.
2709 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2710 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2711 emitDispatchForLoopBounds);
2713 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2714 emitEmptyBoundParameters);
2717 void CodeGenFunction::EmitOMPParallelSectionsDirective(
2718 const OMPParallelSectionsDirective &S) {
2719 // Emit directive as a combined directive that consists of two implicit
2720 // directives: 'parallel' with 'sections' directive.
2721 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2722 CGF.EmitSections(S);
2724 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2725 emitEmptyBoundParameters);
2728 void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
2729 const RegionCodeGenTy &BodyGen,
2730 const TaskGenTy &TaskGen,
2731 OMPTaskDataTy &Data) {
2732 // Emit outlined function for task construct.
2733 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2734 auto *I = CS->getCapturedDecl()->param_begin();
2735 auto *PartId = std::next(I);
2736 auto *TaskT = std::next(I, 4);
2737 // Check if the task is final
2738 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2739 // If the condition constant folds and can be elided, try to avoid emitting
2740 // the condition and the dead arm of the if/else.
2741 auto *Cond = Clause->getCondition();
2743 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2744 Data.Final.setInt(CondConstant);
2746 Data.Final.setPointer(EvaluateExprAsBool(Cond));
2748 // By default the task is not final.
2749 Data.Final.setInt(/*IntVal=*/false);
2751 // Check if the task has 'priority' clause.
2752 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2753 auto *Prio = Clause->getPriority();
2754 Data.Priority.setInt(/*IntVal=*/true);
2755 Data.Priority.setPointer(EmitScalarConversion(
2756 EmitScalarExpr(Prio), Prio->getType(),
2757 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2758 Prio->getExprLoc()));
2760 // The first function argument for tasks is a thread id, the second one is a
2761 // part id (0 for tied tasks, >=0 for untied task).
2762 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2763 // Get list of private variables.
2764 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2765 auto IRef = C->varlist_begin();
2766 for (auto *IInit : C->private_copies()) {
2767 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2768 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2769 Data.PrivateVars.push_back(*IRef);
2770 Data.PrivateCopies.push_back(IInit);
2775 EmittedAsPrivate.clear();
2776 // Get list of firstprivate variables.
2777 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2778 auto IRef = C->varlist_begin();
2779 auto IElemInitRef = C->inits().begin();
2780 for (auto *IInit : C->private_copies()) {
2781 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2782 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2783 Data.FirstprivateVars.push_back(*IRef);
2784 Data.FirstprivateCopies.push_back(IInit);
2785 Data.FirstprivateInits.push_back(*IElemInitRef);
2791 // Get list of lastprivate variables (for taskloops).
2792 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2793 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2794 auto IRef = C->varlist_begin();
2795 auto ID = C->destination_exprs().begin();
2796 for (auto *IInit : C->private_copies()) {
2797 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2798 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2799 Data.LastprivateVars.push_back(*IRef);
2800 Data.LastprivateCopies.push_back(IInit);
2802 LastprivateDstsOrigs.insert(
2803 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2804 cast<DeclRefExpr>(*IRef)});
2809 SmallVector<const Expr *, 4> LHSs;
2810 SmallVector<const Expr *, 4> RHSs;
2811 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2812 auto IPriv = C->privates().begin();
2813 auto IRed = C->reduction_ops().begin();
2814 auto ILHS = C->lhs_exprs().begin();
2815 auto IRHS = C->rhs_exprs().begin();
2816 for (const auto *Ref : C->varlists()) {
2817 Data.ReductionVars.emplace_back(Ref);
2818 Data.ReductionCopies.emplace_back(*IPriv);
2819 Data.ReductionOps.emplace_back(*IRed);
2820 LHSs.emplace_back(*ILHS);
2821 RHSs.emplace_back(*IRHS);
2822 std::advance(IPriv, 1);
2823 std::advance(IRed, 1);
2824 std::advance(ILHS, 1);
2825 std::advance(IRHS, 1);
2828 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2829 *this, S.getLocStart(), LHSs, RHSs, Data);
2830 // Build list of dependences.
2831 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2832 for (auto *IRef : C->varlists())
2833 Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
2834 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs](
2835 CodeGenFunction &CGF, PrePostActionTy &Action) {
2836 // Set proper addresses for generated private copies.
2837 OMPPrivateScope Scope(CGF);
2838 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2839 !Data.LastprivateVars.empty()) {
2840 enum { PrivatesParam = 2, CopyFnParam = 3 };
2841 auto *CopyFn = CGF.Builder.CreateLoad(
2842 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2843 auto *PrivatesPtr = CGF.Builder.CreateLoad(
2844 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2846 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
2847 llvm::SmallVector<llvm::Value *, 16> CallArgs;
2848 CallArgs.push_back(PrivatesPtr);
2849 for (auto *E : Data.PrivateVars) {
2850 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2851 Address PrivatePtr = CGF.CreateMemTemp(
2852 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2853 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2854 CallArgs.push_back(PrivatePtr.getPointer());
2856 for (auto *E : Data.FirstprivateVars) {
2857 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2858 Address PrivatePtr =
2859 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2860 ".firstpriv.ptr.addr");
2861 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2862 CallArgs.push_back(PrivatePtr.getPointer());
2864 for (auto *E : Data.LastprivateVars) {
2865 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2866 Address PrivatePtr =
2867 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2868 ".lastpriv.ptr.addr");
2869 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2870 CallArgs.push_back(PrivatePtr.getPointer());
2872 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2874 for (auto &&Pair : LastprivateDstsOrigs) {
2875 auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2877 const_cast<VarDecl *>(OrigVD),
2878 /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2880 Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2881 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2882 return CGF.EmitLValue(&DRE).getAddress();
2885 for (auto &&Pair : PrivatePtrs) {
2886 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2887 CGF.getContext().getDeclAlign(Pair.first));
2888 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2891 if (Data.Reductions) {
2892 OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true);
2893 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
2895 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2896 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2897 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2898 RedCG.emitSharedLValue(CGF, Cnt);
2899 RedCG.emitAggregateType(CGF, Cnt);
2900 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2901 CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2903 Address(CGF.EmitScalarConversion(
2904 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2905 CGF.getContext().getPointerType(
2906 Data.ReductionCopies[Cnt]->getType()),
2908 Replacement.getAlignment());
2909 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2910 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2911 [Replacement]() { return Replacement; });
2912 // FIXME: This must removed once the runtime library is fixed.
2913 // Emit required threadprivate variables for
2914 // initilizer/combiner/finalizer.
2915 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2919 // Privatize all private variables except for in_reduction items.
2920 (void)Scope.Privatize();
2921 SmallVector<const Expr *, 4> InRedVars;
2922 SmallVector<const Expr *, 4> InRedPrivs;
2923 SmallVector<const Expr *, 4> InRedOps;
2924 SmallVector<const Expr *, 4> TaskgroupDescriptors;
2925 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2926 auto IPriv = C->privates().begin();
2927 auto IRed = C->reduction_ops().begin();
2928 auto ITD = C->taskgroup_descriptors().begin();
2929 for (const auto *Ref : C->varlists()) {
2930 InRedVars.emplace_back(Ref);
2931 InRedPrivs.emplace_back(*IPriv);
2932 InRedOps.emplace_back(*IRed);
2933 TaskgroupDescriptors.emplace_back(*ITD);
2934 std::advance(IPriv, 1);
2935 std::advance(IRed, 1);
2936 std::advance(ITD, 1);
2939 // Privatize in_reduction items here, because taskgroup descriptors must be
2940 // privatized earlier.
2941 OMPPrivateScope InRedScope(CGF);
2942 if (!InRedVars.empty()) {
2943 ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2944 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2945 RedCG.emitSharedLValue(CGF, Cnt);
2946 RedCG.emitAggregateType(CGF, Cnt);
2947 // The taskgroup descriptor variable is always implicit firstprivate and
2948 // privatized already during procoessing of the firstprivates.
2949 llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(
2950 CGF.EmitLValue(TaskgroupDescriptors[Cnt]), SourceLocation());
2951 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2952 CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2953 Replacement = Address(
2954 CGF.EmitScalarConversion(
2955 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2956 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2958 Replacement.getAlignment());
2959 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2960 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2961 [Replacement]() { return Replacement; });
2962 // FIXME: This must removed once the runtime library is fixed.
2963 // Emit required threadprivate variables for
2964 // initilizer/combiner/finalizer.
2965 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2969 (void)InRedScope.Privatize();
2974 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2975 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2976 Data.NumberOfParts);
2977 OMPLexicalScope Scope(*this, S);
2978 TaskGen(*this, OutlinedFn, Data);
2981 static ImplicitParamDecl *
2982 createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
2983 QualType Ty, CapturedDecl *CD) {
2984 auto *OrigVD = ImplicitParamDecl::Create(
2985 C, CD, SourceLocation(), /*Id=*/nullptr, Ty, ImplicitParamDecl::Other);
2987 DeclRefExpr::Create(C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
2988 /*RefersToEnclosingVariableOrCapture=*/false,
2989 SourceLocation(), Ty, VK_LValue);
2990 auto *PrivateVD = ImplicitParamDecl::Create(
2991 C, CD, SourceLocation(), /*Id=*/nullptr, Ty, ImplicitParamDecl::Other);
2992 auto *PrivateRef = DeclRefExpr::Create(
2993 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
2994 /*RefersToEnclosingVariableOrCapture=*/false, SourceLocation(), Ty,
2996 QualType ElemType = C.getBaseElementType(Ty);
2998 ImplicitParamDecl::Create(C, CD, SourceLocation(), /*Id=*/nullptr,
2999 ElemType, ImplicitParamDecl::Other);
3001 DeclRefExpr::Create(C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3002 /*RefersToEnclosingVariableOrCapture=*/false,
3003 SourceLocation(), ElemType, VK_LValue);
3004 PrivateVD->setInitStyle(VarDecl::CInit);
3005 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3006 InitRef, /*BasePath=*/nullptr,
3008 Data.FirstprivateVars.emplace_back(OrigRef);
3009 Data.FirstprivateCopies.emplace_back(PrivateRef);
3010 Data.FirstprivateInits.emplace_back(InitRef);
3014 void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
3015 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3016 OMPTargetDataInfo &InputInfo) {
3017 // Emit outlined function for task construct.
3018 auto CS = S.getCapturedStmt(OMPD_task);
3019 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3020 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3021 auto *I = CS->getCapturedDecl()->param_begin();
3022 auto *PartId = std::next(I);
3023 auto *TaskT = std::next(I, 4);
3025 // The task is not final.
3026 Data.Final.setInt(/*IntVal=*/false);
3027 // Get list of firstprivate variables.
3028 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3029 auto IRef = C->varlist_begin();
3030 auto IElemInitRef = C->inits().begin();
3031 for (auto *IInit : C->private_copies()) {
3032 Data.FirstprivateVars.push_back(*IRef);
3033 Data.FirstprivateCopies.push_back(IInit);
3034 Data.FirstprivateInits.push_back(*IElemInitRef);
3039 OMPPrivateScope TargetScope(*this);
3040 VarDecl *BPVD = nullptr;
3041 VarDecl *PVD = nullptr;
3042 VarDecl *SVD = nullptr;
3043 if (InputInfo.NumberOfTargetItems > 0) {
3044 auto *CD = CapturedDecl::Create(
3045 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3046 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3047 QualType BaseAndPointersType = getContext().getConstantArrayType(
3048 getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3049 /*IndexTypeQuals=*/0);
3050 BPVD = createImplicitFirstprivateForType(getContext(), Data,
3051 BaseAndPointersType, CD);
3052 PVD = createImplicitFirstprivateForType(getContext(), Data,
3053 BaseAndPointersType, CD);
3054 QualType SizesType = getContext().getConstantArrayType(
3055 getContext().getSizeType(), ArrSize, ArrayType::Normal,
3056 /*IndexTypeQuals=*/0);
3057 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD);
3058 TargetScope.addPrivate(
3059 BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3060 TargetScope.addPrivate(PVD,
3061 [&InputInfo]() { return InputInfo.PointersArray; });
3062 TargetScope.addPrivate(SVD,
3063 [&InputInfo]() { return InputInfo.SizesArray; });
3065 (void)TargetScope.Privatize();
3066 // Build list of dependences.
3067 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3068 for (auto *IRef : C->varlists())
3069 Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
3070 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3071 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3072 // Set proper addresses for generated private copies.
3073 OMPPrivateScope Scope(CGF);
3074 if (!Data.FirstprivateVars.empty()) {
3075 enum { PrivatesParam = 2, CopyFnParam = 3 };
3076 auto *CopyFn = CGF.Builder.CreateLoad(
3077 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
3078 auto *PrivatesPtr = CGF.Builder.CreateLoad(
3079 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
3081 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3082 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3083 CallArgs.push_back(PrivatesPtr);
3084 for (auto *E : Data.FirstprivateVars) {
3085 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3086 Address PrivatePtr =
3087 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3088 ".firstpriv.ptr.addr");
3089 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
3090 CallArgs.push_back(PrivatePtr.getPointer());
3092 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3094 for (auto &&Pair : PrivatePtrs) {
3095 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3096 CGF.getContext().getDeclAlign(Pair.first));
3097 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3100 // Privatize all private variables except for in_reduction items.
3101 (void)Scope.Privatize();
3102 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3103 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3104 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3105 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3106 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3107 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3110 OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true,
3111 /*EmitPreInitStmt=*/false);
3114 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3115 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3116 Data.NumberOfParts);
3117 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3118 IntegerLiteral IfCond(getContext(), TrueOrFalse,
3119 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3122 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn,
3123 SharedsTy, CapturedStruct, &IfCond, Data);
3126 void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
3127 // Emit outlined function for task construct.
3128 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3129 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3130 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3131 const Expr *IfCond = nullptr;
3132 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3133 if (C->getNameModifier() == OMPD_unknown ||
3134 C->getNameModifier() == OMPD_task) {
3135 IfCond = C->getCondition();
3141 // Check if we should emit tied or untied task.
3142 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3143 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3144 CGF.EmitStmt(CS->getCapturedStmt());
3146 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3147 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3148 const OMPTaskDataTy &Data) {
3149 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
3150 SharedsTy, CapturedStruct, IfCond,
3153 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
3156 void CodeGenFunction::EmitOMPTaskyieldDirective(
3157 const OMPTaskyieldDirective &S) {
3158 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
3161 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
3162 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
3165 void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
3166 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
3169 void CodeGenFunction::EmitOMPTaskgroupDirective(
3170 const OMPTaskgroupDirective &S) {
3171 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3173 if (const Expr *E = S.getReductionRef()) {
3174 SmallVector<const Expr *, 4> LHSs;
3175 SmallVector<const Expr *, 4> RHSs;
3177 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3178 auto IPriv = C->privates().begin();
3179 auto IRed = C->reduction_ops().begin();
3180 auto ILHS = C->lhs_exprs().begin();
3181 auto IRHS = C->rhs_exprs().begin();
3182 for (const auto *Ref : C->varlists()) {
3183 Data.ReductionVars.emplace_back(Ref);
3184 Data.ReductionCopies.emplace_back(*IPriv);
3185 Data.ReductionOps.emplace_back(*IRed);
3186 LHSs.emplace_back(*ILHS);
3187 RHSs.emplace_back(*IRHS);
3188 std::advance(IPriv, 1);
3189 std::advance(IRed, 1);
3190 std::advance(ILHS, 1);
3191 std::advance(IRHS, 1);
3194 llvm::Value *ReductionDesc =
3195 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(),
3197 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3198 CGF.EmitVarDecl(*VD);
3199 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3200 /*Volatile=*/false, E->getType());
3202 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3204 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3205 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
3208 void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
3209 CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
3210 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
3211 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3212 FlushClause->varlist_end());
3215 }(), S.getLocStart());
3218 void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3219 const CodeGenLoopTy &CodeGenLoop,
3221 // Emit the loop iteration variable.
3222 auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3223 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
3224 EmitVarDecl(*IVDecl);
3226 // Emit the iterations count variable.
3227 // If it is not a variable, Sema decided to calculate iterations count on each
3228 // iteration (e.g., it is foldable into a constant).
3229 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3230 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3231 // Emit calculation of the iterations count.
3232 EmitIgnoredExpr(S.getCalcLastIteration());
3235 auto &RT = CGM.getOpenMPRuntime();
3237 bool HasLastprivateClause = false;
3238 // Check pre-condition.
3240 OMPLoopScope PreInitScope(*this, S);
3241 // Skip the entire loop if we don't meet the precondition.
3242 // If the condition constant folds and can be elided, avoid emitting the
3245 llvm::BasicBlock *ContBlock = nullptr;
3246 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3250 auto *ThenBlock = createBasicBlock("omp.precond.then");
3251 ContBlock = createBasicBlock("omp.precond.end");
3252 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3253 getProfileCount(&S));
3254 EmitBlock(ThenBlock);
3255 incrementProfileCounter(&S);
3258 emitAlignedClause(*this, S);
3259 // Emit 'then' code.
3261 // Emit helper vars inits.
3263 LValue LB = EmitOMPHelperVar(
3264 *this, cast<DeclRefExpr>(
3265 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3266 ? S.getCombinedLowerBoundVariable()
3267 : S.getLowerBoundVariable())));
3268 LValue UB = EmitOMPHelperVar(
3269 *this, cast<DeclRefExpr>(
3270 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3271 ? S.getCombinedUpperBoundVariable()
3272 : S.getUpperBoundVariable())));
3274 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3276 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3278 OMPPrivateScope LoopScope(*this);
3279 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3280 // Emit implicit barrier to synchronize threads and avoid data races
3281 // on initialization of firstprivate variables and post-update of
3282 // lastprivate variables.
3283 CGM.getOpenMPRuntime().emitBarrierCall(
3284 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
3285 /*ForceSimpleCall=*/true);
3287 EmitOMPPrivateClause(S, LoopScope);
3288 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3289 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3290 !isOpenMPTeamsDirective(S.getDirectiveKind()))
3291 EmitOMPReductionClauseInit(S, LoopScope);
3292 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3293 EmitOMPPrivateLoopCounters(S, LoopScope);
3294 (void)LoopScope.Privatize();
3296 // Detect the distribute schedule kind and chunk.
3297 llvm::Value *Chunk = nullptr;
3298 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
3299 if (auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3300 ScheduleKind = C->getDistScheduleKind();
3301 if (const auto *Ch = C->getChunkSize()) {
3302 Chunk = EmitScalarExpr(Ch);
3303 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3304 S.getIterationVariable()->getType(),
3308 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3309 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3311 // OpenMP [2.10.8, distribute Construct, Description]
3312 // If dist_schedule is specified, kind must be static. If specified,
3313 // iterations are divided into chunks of size chunk_size, chunks are
3314 // assigned to the teams of the league in a round-robin fashion in the
3315 // order of the team number. When no chunk_size is specified, the
3316 // iteration space is divided into chunks that are approximately equal
3317 // in size, and at most one chunk is distributed to each team of the
3318 // league. The size of the chunks is unspecified in this case.
3319 if (RT.isStaticNonchunked(ScheduleKind,
3320 /* Chunked */ Chunk != nullptr)) {
3321 if (isOpenMPSimdDirective(S.getDirectiveKind()))
3322 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3323 CGOpenMPRuntime::StaticRTInput StaticInit(
3324 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3325 LB.getAddress(), UB.getAddress(), ST.getAddress());
3326 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
3329 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3330 // UB = min(UB, GlobalUB);
3331 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3332 ? S.getCombinedEnsureUpperBound()
3333 : S.getEnsureUpperBound());
3335 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3336 ? S.getCombinedInit()
3339 Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3340 ? S.getCombinedCond()
3343 // for distribute alone, codegen
3344 // while (idx <= UB) { BODY; ++idx; }
3345 // when combined with 'for' (e.g. as in 'distribute parallel for')
3346 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3347 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3348 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3349 CodeGenLoop(CGF, S, LoopExit);
3351 [](CodeGenFunction &) {});
3352 EmitBlock(LoopExit.getBlock());
3353 // Tell the runtime we are done.
3354 RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind());
3356 // Emit the outer loop, which requests its work chunk [LB..UB] from
3357 // runtime and runs the inner loop to process it.
3358 const OMPLoopArguments LoopArguments = {
3359 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3361 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3364 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3365 EmitOMPSimdFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3366 return CGF.Builder.CreateIsNotNull(
3367 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3370 OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3371 if (isOpenMPParallelDirective(S.getDirectiveKind()) &&
3372 isOpenMPSimdDirective(S.getDirectiveKind())) {
3373 ReductionKind = OMPD_parallel_for_simd;
3374 } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3375 ReductionKind = OMPD_parallel_for;
3376 } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3377 ReductionKind = OMPD_simd;
3378 } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3379 S.hasClausesOfKind<OMPReductionClause>()) {
3381 "No reduction clauses is allowed in distribute directive.");
3383 EmitOMPReductionClauseFinal(S, ReductionKind);
3384 // Emit post-update of the reduction variables if IsLastIter != 0.
3385 emitPostUpdateForReductionClause(
3386 *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3387 return CGF.Builder.CreateIsNotNull(
3388 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3390 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3391 if (HasLastprivateClause) {
3392 EmitOMPLastprivateClauseFinal(
3393 S, /*NoFinals=*/false,
3394 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
3398 // We're now done with the loop, so jump to the continuation block.
3400 EmitBranch(ContBlock);
3401 EmitBlock(ContBlock, true);
3406 void CodeGenFunction::EmitOMPDistributeDirective(
3407 const OMPDistributeDirective &S) {
3408 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3410 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3412 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3413 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3416 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3417 const CapturedStmt *S) {
3418 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3419 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3420 CGF.CapturedStmtInfo = &CapStmtInfo;
3421 auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3422 Fn->addFnAttr(llvm::Attribute::NoInline);
3426 void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
3427 if (!S.getAssociatedStmt()) {
3428 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3429 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3432 auto *C = S.getSingleClause<OMPSIMDClause>();
3433 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3434 PrePostActionTy &Action) {
3436 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3437 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3438 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3439 auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3440 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3441 OutlinedFn, CapturedVars);
3445 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3448 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3449 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
3452 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
3453 QualType SrcType, QualType DestType,
3454 SourceLocation Loc) {
3455 assert(CGF.hasScalarEvaluationKind(DestType) &&
3456 "DestType must have scalar evaluation kind.");
3457 assert(!Val.isAggregate() && "Must be a scalar or complex.");
3458 return Val.isScalar()
3459 ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
3461 : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
3465 static CodeGenFunction::ComplexPairTy
3466 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
3467 QualType DestType, SourceLocation Loc) {
3468 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3469 "DestType must have complex evaluation kind.");
3470 CodeGenFunction::ComplexPairTy ComplexVal;
3471 if (Val.isScalar()) {
3472 // Convert the input element to the element type of the complex.
3473 auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3474 auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3475 DestElementType, Loc);
3476 ComplexVal = CodeGenFunction::ComplexPairTy(
3477 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3479 assert(Val.isComplex() && "Must be a scalar or complex.");
3480 auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3481 auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3482 ComplexVal.first = CGF.EmitScalarConversion(
3483 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3484 ComplexVal.second = CGF.EmitScalarConversion(
3485 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3490 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3491 LValue LVal, RValue RVal) {
3492 if (LVal.isGlobalReg()) {
3493 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3495 CGF.EmitAtomicStore(RVal, LVal,
3496 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3497 : llvm::AtomicOrdering::Monotonic,
3498 LVal.isVolatile(), /*IsInit=*/false);
3502 void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3503 QualType RValTy, SourceLocation Loc) {
3504 switch (getEvaluationKind(LVal.getType())) {
3506 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3507 *this, RVal, RValTy, LVal.getType(), Loc)),
3512 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3516 llvm_unreachable("Must be a scalar or complex.");
3520 static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3521 const Expr *X, const Expr *V,
3522 SourceLocation Loc) {
3524 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3525 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3526 LValue XLValue = CGF.EmitLValue(X);
3527 LValue VLValue = CGF.EmitLValue(V);
3528 RValue Res = XLValue.isGlobalReg()
3529 ? CGF.EmitLoadOfLValue(XLValue, Loc)
3530 : CGF.EmitAtomicLoad(
3532 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3533 : llvm::AtomicOrdering::Monotonic,
3534 XLValue.isVolatile());
3535 // OpenMP, 2.12.6, atomic Construct
3536 // Any atomic construct with a seq_cst clause forces the atomically
3537 // performed operation to include an implicit flush operation without a
3540 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3541 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3544 static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3545 const Expr *X, const Expr *E,
3546 SourceLocation Loc) {
3548 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3549 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3550 // OpenMP, 2.12.6, atomic Construct
3551 // Any atomic construct with a seq_cst clause forces the atomically
3552 // performed operation to include an implicit flush operation without a
3555 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3558 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3560 BinaryOperatorKind BO,
3561 llvm::AtomicOrdering AO,
3562 bool IsXLHSInRHSPart) {
3563 auto &Context = CGF.CGM.getContext();
3564 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3565 // expression is simple and atomic is allowed for the given type for the
3567 if (BO == BO_Comma || !Update.isScalar() ||
3568 !Update.getScalarVal()->getType()->isIntegerTy() ||
3569 !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3570 (Update.getScalarVal()->getType() !=
3571 X.getAddress().getElementType())) ||
3572 !X.getAddress().getElementType()->isIntegerTy() ||
3573 !Context.getTargetInfo().hasBuiltinAtomic(
3574 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3575 return std::make_pair(false, RValue::get(nullptr));
3577 llvm::AtomicRMWInst::BinOp RMWOp;
3580 RMWOp = llvm::AtomicRMWInst::Add;
3583 if (!IsXLHSInRHSPart)
3584 return std::make_pair(false, RValue::get(nullptr));
3585 RMWOp = llvm::AtomicRMWInst::Sub;
3588 RMWOp = llvm::AtomicRMWInst::And;
3591 RMWOp = llvm::AtomicRMWInst::Or;
3594 RMWOp = llvm::AtomicRMWInst::Xor;
3597 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3598 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3599 : llvm::AtomicRMWInst::Max)
3600 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3601 : llvm::AtomicRMWInst::UMax);
3604 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3605 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3606 : llvm::AtomicRMWInst::Min)
3607 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3608 : llvm::AtomicRMWInst::UMin);
3611 RMWOp = llvm::AtomicRMWInst::Xchg;
3620 return std::make_pair(false, RValue::get(nullptr));
3639 llvm_unreachable("Unsupported atomic update operation");
3641 auto *UpdateVal = Update.getScalarVal();
3642 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3643 UpdateVal = CGF.Builder.CreateIntCast(
3644 IC, X.getAddress().getElementType(),
3645 X.getType()->hasSignedIntegerRepresentation());
3647 auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3648 return std::make_pair(true, RValue::get(Res));
3651 std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
3652 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3653 llvm::AtomicOrdering AO, SourceLocation Loc,
3654 const llvm::function_ref<RValue(RValue)> &CommonGen) {
3655 // Update expressions are allowed to have the following forms:
3656 // x binop= expr; -> xrval + expr;
3657 // x++, ++x -> xrval + 1;
3658 // x--, --x -> xrval - 1;
3659 // x = x binop expr; -> xrval binop expr
3660 // x = expr Op x; - > expr binop xrval;
3661 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3663 if (X.isGlobalReg()) {
3664 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3666 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3668 // Perform compare-and-swap procedure.
3669 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3675 static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3676 const Expr *X, const Expr *E,
3677 const Expr *UE, bool IsXLHSInRHSPart,
3678 SourceLocation Loc) {
3679 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3680 "Update expr in 'atomic update' must be a binary operator.");
3681 auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3682 // Update expressions are allowed to have the following forms:
3683 // x binop= expr; -> xrval + expr;
3684 // x++, ++x -> xrval + 1;
3685 // x--, --x -> xrval - 1;
3686 // x = x binop expr; -> xrval binop expr
3687 // x = expr Op x; - > expr binop xrval;
3688 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3689 LValue XLValue = CGF.EmitLValue(X);
3690 RValue ExprRValue = CGF.EmitAnyExpr(E);
3691 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3692 : llvm::AtomicOrdering::Monotonic;
3693 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3694 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3695 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3696 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3698 [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
3699 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3700 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3701 return CGF.EmitAnyExpr(UE);
3703 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3704 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3705 // OpenMP, 2.12.6, atomic Construct
3706 // Any atomic construct with a seq_cst clause forces the atomically
3707 // performed operation to include an implicit flush operation without a
3710 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3713 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
3714 QualType SourceType, QualType ResType,
3715 SourceLocation Loc) {
3716 switch (CGF.getEvaluationKind(ResType)) {
3719 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3721 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3722 return RValue::getComplex(Res.first, Res.second);
3727 llvm_unreachable("Must be a scalar or complex.");
3730 static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3731 bool IsPostfixUpdate, const Expr *V,
3732 const Expr *X, const Expr *E,
3733 const Expr *UE, bool IsXLHSInRHSPart,
3734 SourceLocation Loc) {
3735 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3736 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3738 LValue VLValue = CGF.EmitLValue(V);
3739 LValue XLValue = CGF.EmitLValue(X);
3740 RValue ExprRValue = CGF.EmitAnyExpr(E);
3741 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3742 : llvm::AtomicOrdering::Monotonic;
3743 QualType NewVValType;
3745 // 'x' is updated with some additional value.
3746 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3747 "Update expr in 'atomic capture' must be a binary operator.");
3748 auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3749 // Update expressions are allowed to have the following forms:
3750 // x binop= expr; -> xrval + expr;
3751 // x++, ++x -> xrval + 1;
3752 // x--, --x -> xrval - 1;
3753 // x = x binop expr; -> xrval binop expr
3754 // x = expr Op x; - > expr binop xrval;
3755 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3756 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3757 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3758 NewVValType = XRValExpr->getType();
3759 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3760 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3761 IsPostfixUpdate](RValue XRValue) -> RValue {
3762 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3763 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3764 RValue Res = CGF.EmitAnyExpr(UE);
3765 NewVVal = IsPostfixUpdate ? XRValue : Res;
3768 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3769 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3771 // 'atomicrmw' instruction was generated.
3772 if (IsPostfixUpdate) {
3773 // Use old value from 'atomicrmw'.
3774 NewVVal = Res.second;
3776 // 'atomicrmw' does not provide new value, so evaluate it using old
3778 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3779 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3780 NewVVal = CGF.EmitAnyExpr(UE);
3784 // 'x' is simply rewritten with some 'expr'.
3785 NewVValType = X->getType().getNonReferenceType();
3786 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3787 X->getType().getNonReferenceType(), Loc);
3788 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) -> RValue {
3792 // Try to perform atomicrmw xchg, otherwise simple exchange.
3793 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3794 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3797 // 'atomicrmw' instruction was generated.
3798 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3801 // Emit post-update store to 'v' of old/new 'x' value.
3802 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3803 // OpenMP, 2.12.6, atomic Construct
3804 // Any atomic construct with a seq_cst clause forces the atomically
3805 // performed operation to include an implicit flush operation without a
3808 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3811 static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
3812 bool IsSeqCst, bool IsPostfixUpdate,
3813 const Expr *X, const Expr *V, const Expr *E,
3814 const Expr *UE, bool IsXLHSInRHSPart,
3815 SourceLocation Loc) {
3818 EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3821 EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3825 EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3828 EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3829 IsXLHSInRHSPart, Loc);
3833 case OMPC_num_threads:
3835 case OMPC_firstprivate:
3836 case OMPC_lastprivate:
3837 case OMPC_reduction:
3838 case OMPC_task_reduction:
3839 case OMPC_in_reduction:
3849 case OMPC_copyprivate:
3851 case OMPC_proc_bind:
3856 case OMPC_threadprivate:
3858 case OMPC_mergeable:
3863 case OMPC_num_teams:
3864 case OMPC_thread_limit:
3866 case OMPC_grainsize:
3868 case OMPC_num_tasks:
3870 case OMPC_dist_schedule:
3871 case OMPC_defaultmap:
3875 case OMPC_use_device_ptr:
3876 case OMPC_is_device_ptr:
3877 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3881 void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
3882 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3883 OpenMPClauseKind Kind = OMPC_unknown;
3884 for (auto *C : S.clauses()) {
3885 // Find first clause (skip seq_cst clause, if it is first).
3886 if (C->getClauseKind() != OMPC_seq_cst) {
3887 Kind = C->getClauseKind();
3893 S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
3894 if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
3895 enterFullExpression(EWC);
3897 // Processing for statements under 'atomic capture'.
3898 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3899 for (const auto *C : Compound->body()) {
3900 if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3901 enterFullExpression(EWC);
3906 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3907 PrePostActionTy &) {
3908 CGF.EmitStopPoint(CS);
3909 EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3910 S.getV(), S.getExpr(), S.getUpdateExpr(),
3911 S.isXLHSInRHSPart(), S.getLocStart());
3913 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3914 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3917 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
3918 const OMPExecutableDirective &S,
3919 const RegionCodeGenTy &CodeGen) {
3920 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
3921 CodeGenModule &CGM = CGF.CGM;
3922 const CapturedStmt &CS = *S.getCapturedStmt(OMPD_target);
3924 llvm::Function *Fn = nullptr;
3925 llvm::Constant *FnID = nullptr;
3927 const Expr *IfCond = nullptr;
3928 // Check for the at most one if clause associated with the target region.
3929 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3930 if (C->getNameModifier() == OMPD_unknown ||
3931 C->getNameModifier() == OMPD_target) {
3932 IfCond = C->getCondition();
3937 // Check if we have any device clause associated with the directive.
3938 const Expr *Device = nullptr;
3939 if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
3940 Device = C->getDevice();
3943 // Check if we have an if clause whose conditional always evaluates to false
3944 // or if we do not have any targets specified. If so the target region is not
3945 // an offload entry point.
3946 bool IsOffloadEntry = true;
3949 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3950 IsOffloadEntry = false;
3952 if (CGM.getLangOpts().OMPTargetTriples.empty())
3953 IsOffloadEntry = false;
3955 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3956 StringRef ParentName;
3957 // In case we have Ctors/Dtors we use the complete type variant to produce
3958 // the mangling of the device outlined kernel.
3959 if (auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3960 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3961 else if (auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3962 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3965 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
3967 // Emit target region as a standalone region.
3968 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
3969 IsOffloadEntry, CodeGen);
3970 OMPLexicalScope Scope(CGF, S);
3971 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3972 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
3973 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
3977 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
3978 PrePostActionTy &Action) {
3979 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3980 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3981 CGF.EmitOMPPrivateClause(S, PrivateScope);
3982 (void)PrivateScope.Privatize();
3985 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3988 void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3989 StringRef ParentName,
3990 const OMPTargetDirective &S) {
3991 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3992 emitTargetRegion(CGF, S, Action);
3995 llvm::Constant *Addr;
3996 // Emit target region as a standalone region.
3997 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3998 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3999 assert(Fn && Addr && "Target device function emission failed.");
4002 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
4003 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4004 emitTargetRegion(CGF, S, Action);
4006 emitCommonOMPTargetDirective(*this, S, CodeGen);
4009 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
4010 const OMPExecutableDirective &S,
4011 OpenMPDirectiveKind InnermostKind,
4012 const RegionCodeGenTy &CodeGen) {
4013 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4014 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4015 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4017 const OMPNumTeamsClause *NT = S.getSingleClause<OMPNumTeamsClause>();
4018 const OMPThreadLimitClause *TL = S.getSingleClause<OMPThreadLimitClause>();
4020 Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr;
4021 Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr;
4023 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4027 OMPTeamsScope Scope(CGF, S);
4028 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
4029 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4030 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
4034 void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
4035 // Emit teams region as a standalone region.
4036 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4037 OMPPrivateScope PrivateScope(CGF);
4038 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4039 CGF.EmitOMPPrivateClause(S, PrivateScope);
4040 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4041 (void)PrivateScope.Privatize();
4042 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
4043 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4045 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4046 emitPostUpdateForReductionClause(
4047 *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4050 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4051 const OMPTargetTeamsDirective &S) {
4052 auto *CS = S.getCapturedStmt(OMPD_teams);
4054 // Emit teams region as a standalone region.
4055 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4056 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4057 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4058 CGF.EmitOMPPrivateClause(S, PrivateScope);
4059 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4060 (void)PrivateScope.Privatize();
4062 CGF.EmitStmt(CS->getCapturedStmt());
4063 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4065 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4066 emitPostUpdateForReductionClause(
4067 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4070 void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
4071 CodeGenModule &CGM, StringRef ParentName,
4072 const OMPTargetTeamsDirective &S) {
4073 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4074 emitTargetTeamsRegion(CGF, Action, S);
4077 llvm::Constant *Addr;
4078 // Emit target region as a standalone region.
4079 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4080 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4081 assert(Fn && Addr && "Target device function emission failed.");
4084 void CodeGenFunction::EmitOMPTargetTeamsDirective(
4085 const OMPTargetTeamsDirective &S) {
4086 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4087 emitTargetTeamsRegion(CGF, Action, S);
4089 emitCommonOMPTargetDirective(*this, S, CodeGen);
4093 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4094 const OMPTargetTeamsDistributeDirective &S) {
4096 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4097 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4100 // Emit teams region as a standalone region.
4101 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4102 PrePostActionTy &) {
4103 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4104 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4105 (void)PrivateScope.Privatize();
4106 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4108 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4110 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4111 emitPostUpdateForReductionClause(CGF, S,
4112 [](CodeGenFunction &) { return nullptr; });
4115 void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
4116 CodeGenModule &CGM, StringRef ParentName,
4117 const OMPTargetTeamsDistributeDirective &S) {
4118 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4119 emitTargetTeamsDistributeRegion(CGF, Action, S);
4122 llvm::Constant *Addr;
4123 // Emit target region as a standalone region.
4124 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4125 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4126 assert(Fn && Addr && "Target device function emission failed.");
4129 void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
4130 const OMPTargetTeamsDistributeDirective &S) {
4131 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4132 emitTargetTeamsDistributeRegion(CGF, Action, S);
4134 emitCommonOMPTargetDirective(*this, S, CodeGen);
4137 static void emitTargetTeamsDistributeSimdRegion(
4138 CodeGenFunction &CGF, PrePostActionTy &Action,
4139 const OMPTargetTeamsDistributeSimdDirective &S) {
4141 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4142 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4145 // Emit teams region as a standalone region.
4146 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4147 PrePostActionTy &) {
4148 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4149 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4150 (void)PrivateScope.Privatize();
4151 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4153 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4155 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4156 emitPostUpdateForReductionClause(CGF, S,
4157 [](CodeGenFunction &) { return nullptr; });
4160 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4161 CodeGenModule &CGM, StringRef ParentName,
4162 const OMPTargetTeamsDistributeSimdDirective &S) {
4163 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4164 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4167 llvm::Constant *Addr;
4168 // Emit target region as a standalone region.
4169 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4170 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4171 assert(Fn && Addr && "Target device function emission failed.");
4174 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
4175 const OMPTargetTeamsDistributeSimdDirective &S) {
4176 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4177 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4179 emitCommonOMPTargetDirective(*this, S, CodeGen);
4182 void CodeGenFunction::EmitOMPTeamsDistributeDirective(
4183 const OMPTeamsDistributeDirective &S) {
4185 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4186 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4189 // Emit teams region as a standalone region.
4190 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4191 PrePostActionTy &) {
4192 OMPPrivateScope PrivateScope(CGF);
4193 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4194 (void)PrivateScope.Privatize();
4195 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4197 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4199 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4200 emitPostUpdateForReductionClause(*this, S,
4201 [](CodeGenFunction &) { return nullptr; });
4204 void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
4205 const OMPTeamsDistributeSimdDirective &S) {
4206 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4207 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4210 // Emit teams region as a standalone region.
4211 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4212 PrePostActionTy &) {
4213 OMPPrivateScope PrivateScope(CGF);
4214 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4215 (void)PrivateScope.Privatize();
4216 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4218 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4220 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4221 emitPostUpdateForReductionClause(*this, S,
4222 [](CodeGenFunction &) { return nullptr; });
4225 void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
4226 const OMPTeamsDistributeParallelForDirective &S) {
4227 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4228 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4232 // Emit teams region as a standalone region.
4233 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4234 PrePostActionTy &) {
4235 OMPPrivateScope PrivateScope(CGF);
4236 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4237 (void)PrivateScope.Privatize();
4238 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4240 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4242 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4243 emitPostUpdateForReductionClause(*this, S,
4244 [](CodeGenFunction &) { return nullptr; });
4247 void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
4248 const OMPTeamsDistributeParallelForSimdDirective &S) {
4249 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4250 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4254 // Emit teams region as a standalone region.
4255 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4256 PrePostActionTy &) {
4257 OMPPrivateScope PrivateScope(CGF);
4258 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4259 (void)PrivateScope.Privatize();
4260 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4261 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4262 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4264 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4265 emitPostUpdateForReductionClause(*this, S,
4266 [](CodeGenFunction &) { return nullptr; });
4269 void CodeGenFunction::EmitOMPCancellationPointDirective(
4270 const OMPCancellationPointDirective &S) {
4271 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
4272 S.getCancelRegion());
4275 void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
4276 const Expr *IfCond = nullptr;
4277 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4278 if (C->getNameModifier() == OMPD_unknown ||
4279 C->getNameModifier() == OMPD_cancel) {
4280 IfCond = C->getCondition();
4284 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
4285 S.getCancelRegion());
4288 CodeGenFunction::JumpDest
4289 CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
4290 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4291 Kind == OMPD_target_parallel)
4293 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4294 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4295 Kind == OMPD_distribute_parallel_for ||
4296 Kind == OMPD_target_parallel_for ||
4297 Kind == OMPD_teams_distribute_parallel_for ||
4298 Kind == OMPD_target_teams_distribute_parallel_for);
4299 return OMPCancelStack.getExitBlock();
4302 void CodeGenFunction::EmitOMPUseDevicePtrClause(
4303 const OMPClause &NC, OMPPrivateScope &PrivateScope,
4304 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4305 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4306 auto OrigVarIt = C.varlist_begin();
4307 auto InitIt = C.inits().begin();
4308 for (auto PvtVarIt : C.private_copies()) {
4309 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4310 auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4311 auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4313 // In order to identify the right initializer we need to match the
4314 // declaration used by the mapping logic. In some cases we may get
4315 // OMPCapturedExprDecl that refers to the original declaration.
4316 const ValueDecl *MatchingVD = OrigVD;
4317 if (auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4318 // OMPCapturedExprDecl are used to privative fields of the current
4320 auto *ME = cast<MemberExpr>(OED->getInit());
4321 assert(isa<CXXThisExpr>(ME->getBase()) &&
4322 "Base should be the current struct!");
4323 MatchingVD = ME->getMemberDecl();
4326 // If we don't have information about the current list item, move on to
4328 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4329 if (InitAddrIt == CaptureDeviceAddrMap.end())
4332 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
4333 // Initialize the temporary initialization variable with the address we
4334 // get from the runtime library. We have to cast the source address
4335 // because it is always a void *. References are materialized in the
4336 // privatization scope, so the initialization here disregards the fact
4337 // the original variable is a reference.
4339 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4340 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4341 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4342 setAddrOfLocalVar(InitVD, InitAddr);
4344 // Emit private declaration, it will be initialized by the value we
4345 // declaration we just added to the local declarations map.
4348 // The initialization variables reached its purpose in the emission
4349 // ofthe previous declaration, so we don't need it anymore.
4350 LocalDeclMap.erase(InitVD);
4352 // Return the address of the private variable.
4353 return GetAddrOfLocalVar(PvtVD);
4355 assert(IsRegistered && "firstprivate var already registered as private");
4356 // Silence the warning about unused variable.
4364 // Generate the instructions for '#pragma omp target data' directive.
4365 void CodeGenFunction::EmitOMPTargetDataDirective(
4366 const OMPTargetDataDirective &S) {
4367 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4369 // Create a pre/post action to signal the privatization of the device pointer.
4370 // This action can be replaced by the OpenMP runtime code generation to
4371 // deactivate privatization.
4372 bool PrivatizeDevicePointers = false;
4373 class DevicePointerPrivActionTy : public PrePostActionTy {
4374 bool &PrivatizeDevicePointers;
4377 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4378 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4379 void Enter(CodeGenFunction &CGF) override {
4380 PrivatizeDevicePointers = true;
4383 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4385 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4386 CodeGenFunction &CGF, PrePostActionTy &Action) {
4387 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4389 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
4392 // Codegen that selects wheather to generate the privatization code or not.
4393 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4394 &InnermostCodeGen](CodeGenFunction &CGF,
4395 PrePostActionTy &Action) {
4396 RegionCodeGenTy RCG(InnermostCodeGen);
4397 PrivatizeDevicePointers = false;
4399 // Call the pre-action to change the status of PrivatizeDevicePointers if
4403 if (PrivatizeDevicePointers) {
4404 OMPPrivateScope PrivateScope(CGF);
4405 // Emit all instances of the use_device_ptr clause.
4406 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4407 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4408 Info.CaptureDeviceAddrMap);
4409 (void)PrivateScope.Privatize();
4415 // Forward the provided action to the privatization codegen.
4416 RegionCodeGenTy PrivRCG(PrivCodeGen);
4417 PrivRCG.setAction(Action);
4419 // Notwithstanding the body of the region is emitted as inlined directive,
4420 // we don't use an inline scope as changes in the references inside the
4421 // region are expected to be visible outside, so we do not privative them.
4422 OMPLexicalScope Scope(CGF, S);
4423 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4427 RegionCodeGenTy RCG(CodeGen);
4429 // If we don't have target devices, don't bother emitting the data mapping
4431 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4436 // Check if we have any if clause associated with the directive.
4437 const Expr *IfCond = nullptr;
4438 if (auto *C = S.getSingleClause<OMPIfClause>())
4439 IfCond = C->getCondition();
4441 // Check if we have any device clause associated with the directive.
4442 const Expr *Device = nullptr;
4443 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4444 Device = C->getDevice();
4446 // Set the action to signal privatization of device pointers.
4447 RCG.setAction(PrivAction);
4449 // Emit region code.
4450 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4454 void CodeGenFunction::EmitOMPTargetEnterDataDirective(
4455 const OMPTargetEnterDataDirective &S) {
4456 // If we don't have target devices, don't bother emitting the data mapping
4458 if (CGM.getLangOpts().OMPTargetTriples.empty())
4461 // Check if we have any if clause associated with the directive.
4462 const Expr *IfCond = nullptr;
4463 if (auto *C = S.getSingleClause<OMPIfClause>())
4464 IfCond = C->getCondition();
4466 // Check if we have any device clause associated with the directive.
4467 const Expr *Device = nullptr;
4468 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4469 Device = C->getDevice();
4471 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4472 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4475 void CodeGenFunction::EmitOMPTargetExitDataDirective(
4476 const OMPTargetExitDataDirective &S) {
4477 // If we don't have target devices, don't bother emitting the data mapping
4479 if (CGM.getLangOpts().OMPTargetTriples.empty())
4482 // Check if we have any if clause associated with the directive.
4483 const Expr *IfCond = nullptr;
4484 if (auto *C = S.getSingleClause<OMPIfClause>())
4485 IfCond = C->getCondition();
4487 // Check if we have any device clause associated with the directive.
4488 const Expr *Device = nullptr;
4489 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4490 Device = C->getDevice();
4492 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4493 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4496 static void emitTargetParallelRegion(CodeGenFunction &CGF,
4497 const OMPTargetParallelDirective &S,
4498 PrePostActionTy &Action) {
4499 // Get the captured statement associated with the 'parallel' region.
4500 auto *CS = S.getCapturedStmt(OMPD_parallel);
4502 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) {
4503 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4504 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4505 CGF.EmitOMPPrivateClause(S, PrivateScope);
4506 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4507 (void)PrivateScope.Privatize();
4508 // TODO: Add support for clauses.
4509 CGF.EmitStmt(CS->getCapturedStmt());
4510 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4512 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4513 emitEmptyBoundParameters);
4514 emitPostUpdateForReductionClause(
4515 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4518 void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
4519 CodeGenModule &CGM, StringRef ParentName,
4520 const OMPTargetParallelDirective &S) {
4521 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4522 emitTargetParallelRegion(CGF, S, Action);
4525 llvm::Constant *Addr;
4526 // Emit target region as a standalone region.
4527 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4528 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4529 assert(Fn && Addr && "Target device function emission failed.");
4532 void CodeGenFunction::EmitOMPTargetParallelDirective(
4533 const OMPTargetParallelDirective &S) {
4534 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4535 emitTargetParallelRegion(CGF, S, Action);
4537 emitCommonOMPTargetDirective(*this, S, CodeGen);
4540 static void emitTargetParallelForRegion(CodeGenFunction &CGF,
4541 const OMPTargetParallelForDirective &S,
4542 PrePostActionTy &Action) {
4544 // Emit directive as a combined directive that consists of two implicit
4545 // directives: 'parallel' with 'for' directive.
4546 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4547 CodeGenFunction::OMPCancelStackRAII CancelRegion(
4548 CGF, OMPD_target_parallel_for, S.hasCancel());
4549 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4550 emitDispatchForLoopBounds);
4552 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4553 emitEmptyBoundParameters);
4556 void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
4557 CodeGenModule &CGM, StringRef ParentName,
4558 const OMPTargetParallelForDirective &S) {
4559 // Emit SPMD target parallel for region as a standalone region.
4560 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4561 emitTargetParallelForRegion(CGF, S, Action);
4564 llvm::Constant *Addr;
4565 // Emit target region as a standalone region.
4566 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4567 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4568 assert(Fn && Addr && "Target device function emission failed.");
4571 void CodeGenFunction::EmitOMPTargetParallelForDirective(
4572 const OMPTargetParallelForDirective &S) {
4573 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4574 emitTargetParallelForRegion(CGF, S, Action);
4576 emitCommonOMPTargetDirective(*this, S, CodeGen);
4580 emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
4581 const OMPTargetParallelForSimdDirective &S,
4582 PrePostActionTy &Action) {
4584 // Emit directive as a combined directive that consists of two implicit
4585 // directives: 'parallel' with 'for' directive.
4586 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4587 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4588 emitDispatchForLoopBounds);
4590 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4591 emitEmptyBoundParameters);
4594 void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
4595 CodeGenModule &CGM, StringRef ParentName,
4596 const OMPTargetParallelForSimdDirective &S) {
4597 // Emit SPMD target parallel for region as a standalone region.
4598 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4599 emitTargetParallelForSimdRegion(CGF, S, Action);
4602 llvm::Constant *Addr;
4603 // Emit target region as a standalone region.
4604 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4605 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4606 assert(Fn && Addr && "Target device function emission failed.");
4609 void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
4610 const OMPTargetParallelForSimdDirective &S) {
4611 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4612 emitTargetParallelForSimdRegion(CGF, S, Action);
4614 emitCommonOMPTargetDirective(*this, S, CodeGen);
4617 /// Emit a helper variable and return corresponding lvalue.
4618 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4619 const ImplicitParamDecl *PVD,
4620 CodeGenFunction::OMPPrivateScope &Privates) {
4621 auto *VDecl = cast<VarDecl>(Helper->getDecl());
4622 Privates.addPrivate(
4623 VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); });
4626 void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
4627 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
4628 // Emit outlined function for task construct.
4629 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
4630 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
4631 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4632 const Expr *IfCond = nullptr;
4633 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4634 if (C->getNameModifier() == OMPD_unknown ||
4635 C->getNameModifier() == OMPD_taskloop) {
4636 IfCond = C->getCondition();
4642 // Check if taskloop must be emitted without taskgroup.
4643 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
4644 // TODO: Check if we should emit tied or untied task.
4646 // Set scheduling for taskloop
4647 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4649 Data.Schedule.setInt(/*IntVal=*/false);
4650 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4651 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4653 Data.Schedule.setInt(/*IntVal=*/true);
4654 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4657 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4659 // for (IV in 0..LastIteration) BODY;
4660 // <Final counter/linear vars updates>;
4664 // Emit: if (PreCond) - begin.
4665 // If the condition constant folds and can be elided, avoid emitting the
4668 llvm::BasicBlock *ContBlock = nullptr;
4669 OMPLoopScope PreInitScope(CGF, S);
4670 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4674 auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4675 ContBlock = CGF.createBasicBlock("taskloop.if.end");
4676 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4677 CGF.getProfileCount(&S));
4678 CGF.EmitBlock(ThenBlock);
4679 CGF.incrementProfileCounter(&S);
4682 if (isOpenMPSimdDirective(S.getDirectiveKind()))
4683 CGF.EmitOMPSimdInit(S);
4685 OMPPrivateScope LoopScope(CGF);
4686 // Emit helper vars inits.
4687 enum { LowerBound = 5, UpperBound, Stride, LastIter };
4688 auto *I = CS->getCapturedDecl()->param_begin();
4689 auto *LBP = std::next(I, LowerBound);
4690 auto *UBP = std::next(I, UpperBound);
4691 auto *STP = std::next(I, Stride);
4692 auto *LIP = std::next(I, LastIter);
4693 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4695 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4697 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4698 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4700 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4701 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4702 (void)LoopScope.Privatize();
4703 // Emit the loop iteration variable.
4704 const Expr *IVExpr = S.getIterationVariable();
4705 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4706 CGF.EmitVarDecl(*IVDecl);
4707 CGF.EmitIgnoredExpr(S.getInit());
4709 // Emit the iterations count variable.
4710 // If it is not a variable, Sema decided to calculate iterations count on
4711 // each iteration (e.g., it is foldable into a constant).
4712 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4713 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4714 // Emit calculation of the iterations count.
4715 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4718 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4720 [&S](CodeGenFunction &CGF) {
4721 CGF.EmitOMPLoopBody(S, JumpDest());
4722 CGF.EmitStopPoint(&S);
4724 [](CodeGenFunction &) {});
4725 // Emit: if (PreCond) - end.
4727 CGF.EmitBranch(ContBlock);
4728 CGF.EmitBlock(ContBlock, true);
4730 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4731 if (HasLastprivateClause) {
4732 CGF.EmitOMPLastprivateClauseFinal(
4733 S, isOpenMPSimdDirective(S.getDirectiveKind()),
4734 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4735 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4736 (*LIP)->getType(), S.getLocStart())));
4739 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4740 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4741 const OMPTaskDataTy &Data) {
4742 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) {
4743 OMPLoopScope PreInitScope(CGF, S);
4744 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4745 OutlinedFn, SharedsTy,
4746 CapturedStruct, IfCond, Data);
4748 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4752 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4754 CGM.getOpenMPRuntime().emitTaskgroupRegion(
4756 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4757 PrePostActionTy &Action) {
4759 CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4765 void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
4766 EmitOMPTaskLoopBasedDirective(S);
4769 void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
4770 const OMPTaskLoopSimdDirective &S) {
4771 EmitOMPTaskLoopBasedDirective(S);
4774 // Generate the instructions for '#pragma omp target update' directive.
4775 void CodeGenFunction::EmitOMPTargetUpdateDirective(
4776 const OMPTargetUpdateDirective &S) {
4777 // If we don't have target devices, don't bother emitting the data mapping
4779 if (CGM.getLangOpts().OMPTargetTriples.empty())
4782 // Check if we have any if clause associated with the directive.
4783 const Expr *IfCond = nullptr;
4784 if (auto *C = S.getSingleClause<OMPIfClause>())
4785 IfCond = C->getCondition();
4787 // Check if we have any device clause associated with the directive.
4788 const Expr *Device = nullptr;
4789 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4790 Device = C->getDevice();
4792 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4793 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4796 void CodeGenFunction::EmitSimpleOMPExecutableDirective(
4797 const OMPExecutableDirective &D) {
4798 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
4800 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
4801 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
4802 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
4804 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4805 for (const auto *E : LD->counters()) {
4806 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4807 cast<DeclRefExpr>(E)->getDecl())) {
4808 // Emit only those that were not explicitly referenced in clauses.
4809 if (!CGF.LocalDeclMap.count(VD))
4810 CGF.EmitVarDecl(*VD);
4814 const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
4815 while (const auto *CCS = dyn_cast<CapturedStmt>(CS->getCapturedStmt()))
4817 CGF.EmitStmt(CS->getCapturedStmt());
4820 OMPSimdLexicalScope Scope(*this, D);
4821 CGM.getOpenMPRuntime().emitInlinedDirective(
4823 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
4824 : D.getDirectiveKind(),