1 //===----- CGOpenMPRuntimeNVPTX.h - Interface to OpenMP NVPTX Runtimes ----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This provides a class for OpenMP runtime code generation specialized to NVPTX
13 //===----------------------------------------------------------------------===//
15 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
16 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
18 #include "CGOpenMPRuntime.h"
19 #include "CodeGenFunction.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "llvm/IR/CallSite.h"
26 class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntime {
28 /// Defines the execution mode.
30 /// SPMD execution mode (all threads are worker threads).
32 /// Non-SPMD execution mode (1 master thread, others are workers).
34 /// Unknown execution mode (orphaned directive).
38 /// Parallel outlined function work for workers to execute.
39 llvm::SmallVector<llvm::Function *, 16> Work;
41 struct EntryFunctionState {
42 llvm::BasicBlock *ExitBB = nullptr;
45 class WorkerFunctionState {
47 llvm::Function *WorkerFn;
48 const CGFunctionInfo &CGFI;
51 WorkerFunctionState(CodeGenModule &CGM, SourceLocation Loc);
54 void createWorkerFunction(CodeGenModule &CGM);
57 ExecutionMode getExecutionMode() const;
59 bool requiresFullRuntime() const { return RequiresFullRuntime; }
61 /// Get barrier to synchronize all threads in a block.
62 void syncCTAThreads(CodeGenFunction &CGF);
64 /// Emit the worker function for the current target region.
65 void emitWorkerFunction(WorkerFunctionState &WST);
67 /// Helper for worker function. Emit body of worker loop.
68 void emitWorkerLoop(CodeGenFunction &CGF, WorkerFunctionState &WST);
70 /// Helper for non-SPMD target entry function. Guide the master and
71 /// worker threads to their respective locations.
72 void emitNonSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST,
73 WorkerFunctionState &WST);
75 /// Signal termination of OMP execution for non-SPMD target entry
77 void emitNonSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST);
79 /// Helper for generic variables globalization prolog.
80 void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc,
81 bool WithSPMDCheck = false);
83 /// Helper for generic variables globalization epilog.
84 void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false);
86 /// Helper for SPMD mode target directive's entry function.
87 void emitSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST,
88 const OMPExecutableDirective &D);
90 /// Signal termination of SPMD mode execution.
91 void emitSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST);
94 // Base class overrides.
97 /// Creates offloading entry for the provided entry ID \a ID,
98 /// address \a Addr, size \a Size, and flags \a Flags.
99 void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr,
100 uint64_t Size, int32_t Flags,
101 llvm::GlobalValue::LinkageTypes Linkage) override;
103 /// Emit outlined function specialized for the Fork-Join
104 /// programming model for applicable target directives on the NVPTX device.
105 /// \param D Directive to emit.
106 /// \param ParentName Name of the function that encloses the target region.
107 /// \param OutlinedFn Outlined function value to be defined by this call.
108 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
109 /// \param IsOffloadEntry True if the outlined function is an offload entry.
110 /// An outlined function may not be an entry if, e.g. the if clause always
111 /// evaluates to false.
112 void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
113 llvm::Function *&OutlinedFn,
114 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
115 const RegionCodeGenTy &CodeGen);
117 /// Emit outlined function specialized for the Single Program
118 /// Multiple Data programming model for applicable target directives on the
120 /// \param D Directive to emit.
121 /// \param ParentName Name of the function that encloses the target region.
122 /// \param OutlinedFn Outlined function value to be defined by this call.
123 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
124 /// \param IsOffloadEntry True if the outlined function is an offload entry.
125 /// \param CodeGen Object containing the target statements.
126 /// An outlined function may not be an entry if, e.g. the if clause always
127 /// evaluates to false.
128 void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
129 llvm::Function *&OutlinedFn,
130 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
131 const RegionCodeGenTy &CodeGen);
133 /// Emit outlined function for 'target' directive on the NVPTX
135 /// \param D Directive to emit.
136 /// \param ParentName Name of the function that encloses the target region.
137 /// \param OutlinedFn Outlined function value to be defined by this call.
138 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
139 /// \param IsOffloadEntry True if the outlined function is an offload entry.
140 /// An outlined function may not be an entry if, e.g. the if clause always
141 /// evaluates to false.
142 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
143 StringRef ParentName,
144 llvm::Function *&OutlinedFn,
145 llvm::Constant *&OutlinedFnID,
147 const RegionCodeGenTy &CodeGen) override;
149 /// Emits code for parallel or serial call of the \a OutlinedFn with
150 /// variables captured in a record which address is stored in \a
152 /// This call is for the Non-SPMD Execution Mode.
153 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
154 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
155 /// \param CapturedVars A pointer to the record with the references to
156 /// variables used in \a OutlinedFn function.
157 /// \param IfCond Condition in the associated 'if' clause, if it was
158 /// specified, nullptr otherwise.
159 void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
160 llvm::Value *OutlinedFn,
161 ArrayRef<llvm::Value *> CapturedVars,
164 /// Emits code for parallel or serial call of the \a OutlinedFn with
165 /// variables captured in a record which address is stored in \a
167 /// This call is for a parallel directive within an SPMD target directive.
168 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
169 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
170 /// \param CapturedVars A pointer to the record with the references to
171 /// variables used in \a OutlinedFn function.
172 /// \param IfCond Condition in the associated 'if' clause, if it was
173 /// specified, nullptr otherwise.
175 void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
176 llvm::Value *OutlinedFn,
177 ArrayRef<llvm::Value *> CapturedVars,
181 /// Get the function name of an outlined region.
182 // The name can be customized depending on the target.
184 StringRef getOutlinedHelperName() const override {
185 return "__omp_outlined__";
188 /// Check if the default location must be constant.
189 /// Constant for NVPTX for better optimization.
190 bool isDefaultLocationConstant() const override { return true; }
192 /// Returns additional flags that can be stored in reserved_2 field of the
193 /// default location.
194 /// For NVPTX target contains data about SPMD/Non-SPMD execution mode +
195 /// Full/Lightweight runtime mode. Used for better optimization.
196 unsigned getDefaultLocationReserved2Flags() const override;
199 explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM);
200 void clear() override;
202 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
203 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
204 virtual void emitProcBindClause(CodeGenFunction &CGF,
205 OpenMPProcBindClauseKind ProcBind,
206 SourceLocation Loc) override;
208 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
209 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
211 /// \param NumThreads An integer value of threads.
212 virtual void emitNumThreadsClause(CodeGenFunction &CGF,
213 llvm::Value *NumThreads,
214 SourceLocation Loc) override;
216 /// This function ought to emit, in the general case, a call to
217 // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed
218 // as these numbers are obtained through the PTX grid and block configuration.
219 /// \param NumTeams An integer expression of teams.
220 /// \param ThreadLimit An integer expression of threads.
221 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
222 const Expr *ThreadLimit, SourceLocation Loc) override;
224 /// Emits inlined function for the specified OpenMP parallel
226 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
227 /// kmp_int32 BoundID, struct context_vars*).
228 /// \param D OpenMP directive.
229 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
230 /// \param InnermostKind Kind of innermost directive (for simple directives it
231 /// is a directive itself, for combined - its innermost directive).
232 /// \param CodeGen Code generation sequence for the \a D directive.
234 emitParallelOutlinedFunction(const OMPExecutableDirective &D,
235 const VarDecl *ThreadIDVar,
236 OpenMPDirectiveKind InnermostKind,
237 const RegionCodeGenTy &CodeGen) override;
239 /// Emits inlined function for the specified OpenMP teams
241 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
242 /// kmp_int32 BoundID, struct context_vars*).
243 /// \param D OpenMP directive.
244 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
245 /// \param InnermostKind Kind of innermost directive (for simple directives it
246 /// is a directive itself, for combined - its innermost directive).
247 /// \param CodeGen Code generation sequence for the \a D directive.
249 emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
250 const VarDecl *ThreadIDVar,
251 OpenMPDirectiveKind InnermostKind,
252 const RegionCodeGenTy &CodeGen) override;
254 /// Emits code for teams call of the \a OutlinedFn with
255 /// variables captured in a record which address is stored in \a
257 /// \param OutlinedFn Outlined function to be run by team masters. Type of
258 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
259 /// \param CapturedVars A pointer to the record with the references to
260 /// variables used in \a OutlinedFn function.
262 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
263 SourceLocation Loc, llvm::Value *OutlinedFn,
264 ArrayRef<llvm::Value *> CapturedVars) override;
266 /// Emits code for parallel or serial call of the \a OutlinedFn with
267 /// variables captured in a record which address is stored in \a
269 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
270 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
271 /// \param CapturedVars A pointer to the record with the references to
272 /// variables used in \a OutlinedFn function.
273 /// \param IfCond Condition in the associated 'if' clause, if it was
274 /// specified, nullptr otherwise.
275 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
276 llvm::Value *OutlinedFn,
277 ArrayRef<llvm::Value *> CapturedVars,
278 const Expr *IfCond) override;
280 /// Emit an implicit/explicit barrier for OpenMP threads.
281 /// \param Kind Directive for which this implicit barrier call must be
282 /// generated. Must be OMPD_barrier for explicit barrier generation.
283 /// \param EmitChecks true if need to emit checks for cancellation barriers.
284 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
285 /// runtime class decides which one to emit (simple or with cancellation
288 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
289 OpenMPDirectiveKind Kind, bool EmitChecks = true,
290 bool ForceSimpleCall = false) override;
292 /// Emits a critical region.
293 /// \param CriticalName Name of the critical region.
294 /// \param CriticalOpGen Generator for the statement associated with the given
296 /// \param Hint Value of the 'hint' clause (optional).
297 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
298 const RegionCodeGenTy &CriticalOpGen,
300 const Expr *Hint = nullptr) override;
302 /// Emit a code for reduction clause.
304 /// \param Privates List of private copies for original reduction arguments.
305 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
306 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
307 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
308 /// or 'operator binop(LHS, RHS)'.
309 /// \param Options List of options for reduction codegen:
310 /// WithNowait true if parent directive has also nowait clause, false
312 /// SimpleReduction Emit reduction operation only. Used for omp simd
313 /// directive on the host.
314 /// ReductionKind The kind of reduction to perform.
315 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
316 ArrayRef<const Expr *> Privates,
317 ArrayRef<const Expr *> LHSExprs,
318 ArrayRef<const Expr *> RHSExprs,
319 ArrayRef<const Expr *> ReductionOps,
320 ReductionOptionsTy Options) override;
322 /// Returns specified OpenMP runtime function for the current OpenMP
323 /// implementation. Specialized for the NVPTX device.
324 /// \param Function OpenMP runtime function.
325 /// \return Specified function.
326 llvm::Constant *createNVPTXRuntimeFunction(unsigned Function);
328 /// Translates the native parameter of outlined function if this is required
330 /// \param FD Field decl from captured record for the parameter.
331 /// \param NativeParam Parameter itself.
332 const VarDecl *translateParameter(const FieldDecl *FD,
333 const VarDecl *NativeParam) const override;
335 /// Gets the address of the native argument basing on the address of the
336 /// target-specific parameter.
337 /// \param NativeParam Parameter itself.
338 /// \param TargetParam Corresponding target-specific parameter.
339 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
340 const VarDecl *TargetParam) const override;
342 /// Emits call of the outlined function with the provided arguments,
343 /// translating these arguments to correct target-specific arguments.
344 void emitOutlinedFunctionCall(
345 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
346 ArrayRef<llvm::Value *> Args = llvm::None) const override;
348 /// Emits OpenMP-specific function prolog.
349 /// Required for device constructs.
350 void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;
352 /// Gets the OpenMP-specific address of the local variable.
353 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
354 const VarDecl *VD) override;
356 /// Target codegen is specialized based on two data-sharing modes: CUDA, in
357 /// which the local variables are actually global threadlocal, and Generic, in
358 /// which the local variables are placed in global memory if they may escape
359 /// their declaration context.
360 enum DataSharingMode {
361 /// CUDA data sharing mode.
363 /// Generic data-sharing mode.
367 /// Cleans up references to the objects in finished function.
369 void functionFinished(CodeGenFunction &CGF) override;
371 /// Choose a default value for the dist_schedule clause.
372 void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
373 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
374 llvm::Value *&Chunk) const override;
376 /// Choose a default value for the schedule clause.
377 void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
378 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
379 const Expr *&ChunkExpr) const override;
381 /// Adjust some parameters for the target-based directives, like addresses of
382 /// the variables captured by reference in lambdas.
383 void adjustTargetSpecificDataForLambdas(
384 CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;
386 /// Perform check on requires decl to ensure that target architecture
387 /// supports unified addressing
388 void checkArchForUnifiedAddressing(CodeGenModule &CGM,
389 const OMPRequiresDecl *D) const override;
392 /// Track the execution mode when codegening directives within a target
393 /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the
394 /// target region and used by containing directives such as 'parallel'
395 /// to emit optimized code.
396 ExecutionMode CurrentExecutionMode = EM_Unknown;
398 /// Check if the full runtime is required (default - yes).
399 bool RequiresFullRuntime = true;
401 /// true if we're emitting the code for the target region and next parallel
402 /// region is L0 for sure.
403 bool IsInTargetMasterThreadRegion = false;
404 /// true if currently emitting code for target/teams/distribute region, false
406 bool IsInTTDRegion = false;
407 /// true if we're definitely in the parallel region.
408 bool IsInParallelRegion = false;
410 /// Map between an outlined function and its wrapper.
411 llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap;
413 /// Emit function which wraps the outline parallel region
414 /// and controls the parameters which are passed to this function.
415 /// The wrapper ensures that the outlined function is called
416 /// with the correct arguments when data is shared.
417 llvm::Function *createParallelDataSharingWrapper(
418 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);
420 /// The data for the single globalized variable.
421 struct MappedVarData {
422 /// Corresponding field in the global record.
423 const FieldDecl *FD = nullptr;
424 /// Corresponding address.
425 Address PrivateAddr = Address::invalid();
426 /// true, if only one element is required (for latprivates in SPMD mode),
427 /// false, if need to create based on the warp-size.
428 bool IsOnePerTeam = false;
429 MappedVarData() = delete;
430 MappedVarData(const FieldDecl *FD, bool IsOnePerTeam = false)
431 : FD(FD), IsOnePerTeam(IsOnePerTeam) {}
433 /// The map of local variables to their addresses in the global memory.
434 using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
435 /// Set of the parameters passed by value escaping OpenMP context.
436 using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;
437 struct FunctionData {
438 DeclToAddrMapTy LocalVarData;
439 llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None;
440 EscapedParamsTy EscapedParameters;
441 llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
442 llvm::SmallVector<llvm::Value *, 4> EscapedVariableLengthDeclsAddrs;
443 const RecordDecl *GlobalRecord = nullptr;
444 llvm::Optional<const RecordDecl *> SecondaryGlobalRecord = llvm::None;
445 llvm::Value *GlobalRecordAddr = nullptr;
446 llvm::Value *IsInSPMDModeFlag = nullptr;
447 std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
449 /// Maps the function to the list of the globalized variables with their
451 llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
452 /// List of records for the globalized variables in target/teams/distribute
453 /// contexts. Inner records are going to be joined into the single record,
454 /// while those resulting records are going to be joined into the single
455 /// union. This resulting union (one per CU) is the entry point for the static
456 /// memory management runtime functions.
457 struct GlobalPtrSizeRecsTy {
458 llvm::GlobalVariable *UseSharedMemory = nullptr;
459 llvm::GlobalVariable *RecSize = nullptr;
460 llvm::GlobalVariable *Buffer = nullptr;
462 llvm::SmallVector<const RecordDecl *, 2> Records;
463 unsigned RegionCounter = 0;
465 llvm::SmallVector<GlobalPtrSizeRecsTy, 8> GlobalizedRecords;
466 /// Shared pointer for the global memory in the global memory buffer used for
467 /// the given kernel.
468 llvm::GlobalVariable *KernelStaticGlobalized = nullptr;
469 /// Pair of the Non-SPMD team and all reductions variables in this team
471 std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>>
475 } // CodeGen namespace.
476 } // clang namespace.
478 #endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H