1 //===---- CGOpenMPRuntimeNVPTX.cpp - Interface to OpenMP NVPTX Runtimes ---===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This provides a class for OpenMP runtime code generation specialized to NVPTX
13 //===----------------------------------------------------------------------===//
15 #include "CGOpenMPRuntimeNVPTX.h"
16 #include "clang/AST/DeclOpenMP.h"
17 #include "CodeGenFunction.h"
18 #include "clang/AST/StmtOpenMP.h"
20 using namespace clang;
21 using namespace CodeGen;
24 enum OpenMPRTLFunctionNVPTX {
25 /// \brief Call to void __kmpc_kernel_init(kmp_int32 thread_limit);
26 OMPRTL_NVPTX__kmpc_kernel_init,
27 /// \brief Call to void __kmpc_kernel_deinit();
28 OMPRTL_NVPTX__kmpc_kernel_deinit,
29 /// \brief Call to void __kmpc_kernel_prepare_parallel(void
30 /// *outlined_function);
31 OMPRTL_NVPTX__kmpc_kernel_prepare_parallel,
32 /// \brief Call to bool __kmpc_kernel_parallel(void **outlined_function);
33 OMPRTL_NVPTX__kmpc_kernel_parallel,
34 /// \brief Call to void __kmpc_kernel_end_parallel();
35 OMPRTL_NVPTX__kmpc_kernel_end_parallel,
36 /// Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
38 OMPRTL_NVPTX__kmpc_serialized_parallel,
39 /// Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
41 OMPRTL_NVPTX__kmpc_end_serialized_parallel,
44 /// Pre(post)-action for different OpenMP constructs specialized for NVPTX.
45 class NVPTXActionTy final : public PrePostActionTy {
46 llvm::Value *EnterCallee;
47 ArrayRef<llvm::Value *> EnterArgs;
48 llvm::Value *ExitCallee;
49 ArrayRef<llvm::Value *> ExitArgs;
51 llvm::BasicBlock *ContBlock = nullptr;
54 NVPTXActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs,
55 llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs,
56 bool Conditional = false)
57 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
58 ExitArgs(ExitArgs), Conditional(Conditional) {}
59 void Enter(CodeGenFunction &CGF) override {
60 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
62 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
63 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
64 ContBlock = CGF.createBasicBlock("omp_if.end");
65 // Generate the branch (If-stmt)
66 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
67 CGF.EmitBlock(ThenBlock);
70 void Done(CodeGenFunction &CGF) {
71 // Emit the rest of blocks/branches
72 CGF.EmitBranch(ContBlock);
73 CGF.EmitBlock(ContBlock, true);
75 void Exit(CodeGenFunction &CGF) override {
76 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
79 } // anonymous namespace
81 /// Get the GPU warp size.
82 static llvm::Value *getNVPTXWarpSize(CodeGenFunction &CGF) {
83 CGBuilderTy &Bld = CGF.Builder;
84 return Bld.CreateCall(
85 llvm::Intrinsic::getDeclaration(
86 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_warpsize),
87 llvm::None, "nvptx_warp_size");
90 /// Get the id of the current thread on the GPU.
91 static llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF) {
92 CGBuilderTy &Bld = CGF.Builder;
93 return Bld.CreateCall(
94 llvm::Intrinsic::getDeclaration(
95 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x),
96 llvm::None, "nvptx_tid");
99 /// Get the maximum number of threads in a block of the GPU.
100 static llvm::Value *getNVPTXNumThreads(CodeGenFunction &CGF) {
101 CGBuilderTy &Bld = CGF.Builder;
102 return Bld.CreateCall(
103 llvm::Intrinsic::getDeclaration(
104 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x),
105 llvm::None, "nvptx_num_threads");
108 /// Get barrier to synchronize all threads in a block.
109 static void getNVPTXCTABarrier(CodeGenFunction &CGF) {
110 CGBuilderTy &Bld = CGF.Builder;
111 Bld.CreateCall(llvm::Intrinsic::getDeclaration(
112 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_barrier0));
115 /// Synchronize all GPU threads in a block.
116 static void syncCTAThreads(CodeGenFunction &CGF) { getNVPTXCTABarrier(CGF); }
118 /// Get the value of the thread_limit clause in the teams directive.
119 /// The runtime encodes thread_limit in the launch parameter, always starting
120 /// thread_limit+warpSize threads per team.
121 static llvm::Value *getThreadLimit(CodeGenFunction &CGF) {
122 CGBuilderTy &Bld = CGF.Builder;
123 return Bld.CreateSub(getNVPTXNumThreads(CGF), getNVPTXWarpSize(CGF),
127 /// Get the thread id of the OMP master thread.
128 /// The master thread id is the first thread (lane) of the last warp in the
129 /// GPU block. Warp size is assumed to be some power of 2.
130 /// Thread id is 0 indexed.
131 /// E.g: If NumThreads is 33, master id is 32.
132 /// If NumThreads is 64, master id is 32.
133 /// If NumThreads is 1024, master id is 992.
134 static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) {
135 CGBuilderTy &Bld = CGF.Builder;
136 llvm::Value *NumThreads = getNVPTXNumThreads(CGF);
138 // We assume that the warp size is a power of 2.
139 llvm::Value *Mask = Bld.CreateSub(getNVPTXWarpSize(CGF), Bld.getInt32(1));
141 return Bld.CreateAnd(Bld.CreateSub(NumThreads, Bld.getInt32(1)),
142 Bld.CreateNot(Mask), "master_tid");
145 CGOpenMPRuntimeNVPTX::WorkerFunctionState::WorkerFunctionState(
147 : WorkerFn(nullptr), CGFI(nullptr) {
148 createWorkerFunction(CGM);
151 void CGOpenMPRuntimeNVPTX::WorkerFunctionState::createWorkerFunction(
152 CodeGenModule &CGM) {
153 // Create an worker function with no arguments.
154 CGFI = &CGM.getTypes().arrangeNullaryFunction();
156 WorkerFn = llvm::Function::Create(
157 CGM.getTypes().GetFunctionType(*CGFI), llvm::GlobalValue::InternalLinkage,
158 /* placeholder */ "_worker", &CGM.getModule());
159 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, WorkerFn, *CGFI);
162 void CGOpenMPRuntimeNVPTX::emitGenericKernel(const OMPExecutableDirective &D,
163 StringRef ParentName,
164 llvm::Function *&OutlinedFn,
165 llvm::Constant *&OutlinedFnID,
167 const RegionCodeGenTy &CodeGen) {
168 EntryFunctionState EST;
169 WorkerFunctionState WST(CGM);
172 // Emit target region as a standalone region.
173 class NVPTXPrePostActionTy : public PrePostActionTy {
174 CGOpenMPRuntimeNVPTX &RT;
175 CGOpenMPRuntimeNVPTX::EntryFunctionState &EST;
176 CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST;
179 NVPTXPrePostActionTy(CGOpenMPRuntimeNVPTX &RT,
180 CGOpenMPRuntimeNVPTX::EntryFunctionState &EST,
181 CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST)
182 : RT(RT), EST(EST), WST(WST) {}
183 void Enter(CodeGenFunction &CGF) override {
184 RT.emitGenericEntryHeader(CGF, EST, WST);
186 void Exit(CodeGenFunction &CGF) override {
187 RT.emitGenericEntryFooter(CGF, EST);
189 } Action(*this, EST, WST);
190 CodeGen.setAction(Action);
191 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
192 IsOffloadEntry, CodeGen);
194 // Create the worker function
195 emitWorkerFunction(WST);
197 // Now change the name of the worker function to correspond to this target
198 // region's entry function.
199 WST.WorkerFn->setName(OutlinedFn->getName() + "_worker");
202 // Setup NVPTX threads for master-worker OpenMP scheme.
203 void CGOpenMPRuntimeNVPTX::emitGenericEntryHeader(CodeGenFunction &CGF,
204 EntryFunctionState &EST,
205 WorkerFunctionState &WST) {
206 CGBuilderTy &Bld = CGF.Builder;
208 llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker");
209 llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck");
210 llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master");
211 EST.ExitBB = CGF.createBasicBlock(".exit");
214 Bld.CreateICmpULT(getNVPTXThreadID(CGF), getThreadLimit(CGF));
215 Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB);
217 CGF.EmitBlock(WorkerBB);
218 CGF.EmitCallOrInvoke(WST.WorkerFn, llvm::None);
219 CGF.EmitBranch(EST.ExitBB);
221 CGF.EmitBlock(MasterCheckBB);
223 Bld.CreateICmpEQ(getNVPTXThreadID(CGF), getMasterThreadID(CGF));
224 Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB);
226 CGF.EmitBlock(MasterBB);
227 // First action in sequential region:
228 // Initialize the state of the OpenMP runtime library on the GPU.
229 llvm::Value *Args[] = {getThreadLimit(CGF)};
231 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_init), Args);
234 void CGOpenMPRuntimeNVPTX::emitGenericEntryFooter(CodeGenFunction &CGF,
235 EntryFunctionState &EST) {
237 EST.ExitBB = CGF.createBasicBlock(".exit");
239 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier");
240 CGF.EmitBranch(TerminateBB);
242 CGF.EmitBlock(TerminateBB);
243 // Signal termination condition.
245 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_deinit), None);
246 // Barrier to terminate worker threads.
248 // Master thread jumps to exit point.
249 CGF.EmitBranch(EST.ExitBB);
251 CGF.EmitBlock(EST.ExitBB);
252 EST.ExitBB = nullptr;
255 void CGOpenMPRuntimeNVPTX::emitWorkerFunction(WorkerFunctionState &WST) {
256 auto &Ctx = CGM.getContext();
258 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
259 CGF.disableDebugInfo();
260 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, *WST.CGFI, {});
261 emitWorkerLoop(CGF, WST);
262 CGF.FinishFunction();
265 void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF,
266 WorkerFunctionState &WST) {
268 // The workers enter this loop and wait for parallel work from the master.
269 // When the master encounters a parallel region it sets up the work + variable
270 // arguments, and wakes up the workers. The workers first check to see if
271 // they are required for the parallel region, i.e., within the # of requested
272 // parallel threads. The activated workers load the variable arguments and
273 // execute the parallel work.
276 CGBuilderTy &Bld = CGF.Builder;
278 llvm::BasicBlock *AwaitBB = CGF.createBasicBlock(".await.work");
279 llvm::BasicBlock *SelectWorkersBB = CGF.createBasicBlock(".select.workers");
280 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute.parallel");
281 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".terminate.parallel");
282 llvm::BasicBlock *BarrierBB = CGF.createBasicBlock(".barrier.parallel");
283 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
285 CGF.EmitBranch(AwaitBB);
287 // Workers wait for work from master.
288 CGF.EmitBlock(AwaitBB);
289 // Wait for parallel work
293 CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn");
295 CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status");
296 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0));
297 CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy));
299 llvm::Value *Args[] = {WorkFn.getPointer()};
300 llvm::Value *Ret = CGF.EmitRuntimeCall(
301 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_parallel), Args);
302 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
304 // On termination condition (workid == 0), exit loop.
305 llvm::Value *ShouldTerminate =
306 Bld.CreateIsNull(Bld.CreateLoad(WorkFn), "should_terminate");
307 Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB);
309 // Activate requested workers.
310 CGF.EmitBlock(SelectWorkersBB);
311 llvm::Value *IsActive =
312 Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active");
313 Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB);
315 // Signal start of parallel region.
316 CGF.EmitBlock(ExecuteBB);
318 // Process work items: outlined parallel functions.
319 for (auto *W : Work) {
320 // Try to match this outlined function.
321 auto *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy);
323 llvm::Value *WorkFnMatch =
324 Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match");
326 llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn");
327 llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next");
328 Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB);
330 // Execute this outlined function.
331 CGF.EmitBlock(ExecuteFNBB);
333 // Insert call to work function.
334 // FIXME: Pass arguments to outlined function from master thread.
335 auto *Fn = cast<llvm::Function>(W);
337 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".zero.addr");
338 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C=*/0));
339 llvm::Value *FnArgs[] = {ZeroAddr.getPointer(), ZeroAddr.getPointer()};
340 CGF.EmitCallOrInvoke(Fn, FnArgs);
342 // Go to end of parallel region.
343 CGF.EmitBranch(TerminateBB);
345 CGF.EmitBlock(CheckNextBB);
348 // Signal end of parallel region.
349 CGF.EmitBlock(TerminateBB);
351 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_end_parallel),
353 CGF.EmitBranch(BarrierBB);
355 // All active and inactive workers wait at a barrier after parallel region.
356 CGF.EmitBlock(BarrierBB);
357 // Barrier after parallel region.
359 CGF.EmitBranch(AwaitBB);
361 // Exit target region.
362 CGF.EmitBlock(ExitBB);
365 /// \brief Returns specified OpenMP runtime function for the current OpenMP
366 /// implementation. Specialized for the NVPTX device.
367 /// \param Function OpenMP runtime function.
368 /// \return Specified function.
370 CGOpenMPRuntimeNVPTX::createNVPTXRuntimeFunction(unsigned Function) {
371 llvm::Constant *RTLFn = nullptr;
372 switch (static_cast<OpenMPRTLFunctionNVPTX>(Function)) {
373 case OMPRTL_NVPTX__kmpc_kernel_init: {
374 // Build void __kmpc_kernel_init(kmp_int32 thread_limit);
375 llvm::Type *TypeParams[] = {CGM.Int32Ty};
376 llvm::FunctionType *FnTy =
377 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
378 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_init");
381 case OMPRTL_NVPTX__kmpc_kernel_deinit: {
382 // Build void __kmpc_kernel_deinit();
383 llvm::FunctionType *FnTy =
384 llvm::FunctionType::get(CGM.VoidTy, llvm::None, /*isVarArg*/ false);
385 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_deinit");
388 case OMPRTL_NVPTX__kmpc_kernel_prepare_parallel: {
389 /// Build void __kmpc_kernel_prepare_parallel(
390 /// void *outlined_function);
391 llvm::Type *TypeParams[] = {CGM.Int8PtrTy};
392 llvm::FunctionType *FnTy =
393 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
394 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_prepare_parallel");
397 case OMPRTL_NVPTX__kmpc_kernel_parallel: {
398 /// Build bool __kmpc_kernel_parallel(void **outlined_function);
399 llvm::Type *TypeParams[] = {CGM.Int8PtrPtrTy};
400 llvm::Type *RetTy = CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
401 llvm::FunctionType *FnTy =
402 llvm::FunctionType::get(RetTy, TypeParams, /*isVarArg*/ false);
403 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_parallel");
406 case OMPRTL_NVPTX__kmpc_kernel_end_parallel: {
407 /// Build void __kmpc_kernel_end_parallel();
408 llvm::FunctionType *FnTy =
409 llvm::FunctionType::get(CGM.VoidTy, llvm::None, /*isVarArg*/ false);
410 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_end_parallel");
413 case OMPRTL_NVPTX__kmpc_serialized_parallel: {
414 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
416 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
417 llvm::FunctionType *FnTy =
418 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
419 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
422 case OMPRTL_NVPTX__kmpc_end_serialized_parallel: {
423 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
425 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
426 llvm::FunctionType *FnTy =
427 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
428 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
435 void CGOpenMPRuntimeNVPTX::createOffloadEntry(llvm::Constant *ID,
436 llvm::Constant *Addr,
437 uint64_t Size, int32_t) {
438 auto *F = dyn_cast<llvm::Function>(Addr);
439 // TODO: Add support for global variables on the device after declare target
443 llvm::Module *M = F->getParent();
444 llvm::LLVMContext &Ctx = M->getContext();
446 // Get "nvvm.annotations" metadata node
447 llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations");
449 llvm::Metadata *MDVals[] = {
450 llvm::ConstantAsMetadata::get(F), llvm::MDString::get(Ctx, "kernel"),
451 llvm::ConstantAsMetadata::get(
452 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
453 // Append metadata to nvvm.annotations
454 MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
457 void CGOpenMPRuntimeNVPTX::emitTargetOutlinedFunction(
458 const OMPExecutableDirective &D, StringRef ParentName,
459 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
460 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
461 if (!IsOffloadEntry) // Nothing to do.
464 assert(!ParentName.empty() && "Invalid target region parent name!");
466 emitGenericKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
470 CGOpenMPRuntimeNVPTX::CGOpenMPRuntimeNVPTX(CodeGenModule &CGM)
471 : CGOpenMPRuntime(CGM) {
472 if (!CGM.getLangOpts().OpenMPIsDevice)
473 llvm_unreachable("OpenMP NVPTX can only handle device code.");
476 void CGOpenMPRuntimeNVPTX::emitNumTeamsClause(CodeGenFunction &CGF,
477 const Expr *NumTeams,
478 const Expr *ThreadLimit,
479 SourceLocation Loc) {}
481 llvm::Value *CGOpenMPRuntimeNVPTX::emitParallelOrTeamsOutlinedFunction(
482 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
483 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
485 llvm::Function *OutlinedFun = nullptr;
486 if (isa<OMPTeamsDirective>(D)) {
487 llvm::Value *OutlinedFunVal =
488 CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction(
489 D, ThreadIDVar, InnermostKind, CodeGen);
490 OutlinedFun = cast<llvm::Function>(OutlinedFunVal);
491 OutlinedFun->removeFnAttr(llvm::Attribute::NoInline);
492 OutlinedFun->addFnAttr(llvm::Attribute::AlwaysInline);
494 llvm::Value *OutlinedFunVal =
495 CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction(
496 D, ThreadIDVar, InnermostKind, CodeGen);
497 OutlinedFun = cast<llvm::Function>(OutlinedFunVal);
503 void CGOpenMPRuntimeNVPTX::emitTeamsCall(CodeGenFunction &CGF,
504 const OMPExecutableDirective &D,
506 llvm::Value *OutlinedFn,
507 ArrayRef<llvm::Value *> CapturedVars) {
508 if (!CGF.HaveInsertPoint())
512 CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4),
513 /*Name*/ ".zero.addr");
514 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
515 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
516 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
517 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
518 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
519 CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
522 void CGOpenMPRuntimeNVPTX::emitParallelCall(
523 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
524 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) {
525 if (!CGF.HaveInsertPoint())
528 emitGenericParallelCall(CGF, Loc, OutlinedFn, CapturedVars, IfCond);
531 void CGOpenMPRuntimeNVPTX::emitGenericParallelCall(
532 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
533 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) {
534 llvm::Function *Fn = cast<llvm::Function>(OutlinedFn);
536 auto &&L0ParallelGen = [this, Fn, &CapturedVars](CodeGenFunction &CGF,
538 CGBuilderTy &Bld = CGF.Builder;
540 // Prepare for parallel region. Indicate the outlined function.
541 llvm::Value *Args[] = {Bld.CreateBitOrPointerCast(Fn, CGM.Int8PtrTy)};
543 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_prepare_parallel),
546 // Activate workers. This barrier is used by the master to signal
547 // work for the workers.
550 // OpenMP [2.5, Parallel Construct, p.49]
551 // There is an implied barrier at the end of a parallel region. After the
552 // end of a parallel region, only the master thread of the team resumes
553 // execution of the enclosing task region.
555 // The master waits at this barrier until all workers are done.
558 // Remember for post-processing in worker loop.
562 auto *RTLoc = emitUpdateLocation(CGF, Loc);
563 auto *ThreadID = getThreadID(CGF, Loc);
564 llvm::Value *Args[] = {RTLoc, ThreadID};
566 auto &&SeqGen = [this, Fn, &CapturedVars, &Args](CodeGenFunction &CGF,
568 auto &&CodeGen = [this, Fn, &CapturedVars, &Args](CodeGenFunction &CGF,
569 PrePostActionTy &Action) {
572 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
573 OutlinedFnArgs.push_back(
574 llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo()));
575 OutlinedFnArgs.push_back(
576 llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo()));
577 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
578 CGF.EmitCallOrInvoke(Fn, OutlinedFnArgs);
581 RegionCodeGenTy RCG(CodeGen);
582 NVPTXActionTy Action(
583 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_serialized_parallel),
585 createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_end_serialized_parallel),
587 RCG.setAction(Action);
592 emitOMPIfClause(CGF, IfCond, L0ParallelGen, SeqGen);
594 CodeGenFunction::RunCleanupsScope Scope(CGF);
595 RegionCodeGenTy ThenRCG(L0ParallelGen);