1 //===------ CGOpenMPRuntimeGPU.h - Interface to OpenMP GPU Runtimes ------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This provides a generalized class for OpenMP runtime code generation
10 // specialized by GPU targets NVPTX and AMDGCN.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
15 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
17 #include "CGOpenMPRuntime.h"
18 #include "CodeGenFunction.h"
19 #include "clang/AST/StmtOpenMP.h"
20 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
25 class CGOpenMPRuntimeGPU : public CGOpenMPRuntime {
27 /// Defines the execution mode.
29 /// SPMD execution mode (all threads are worker threads).
31 /// Non-SPMD execution mode (1 master thread, others are workers).
33 /// Unknown execution mode (orphaned directive).
37 /// Parallel outlined function work for workers to execute.
38 llvm::SmallVector<llvm::Function *, 16> Work;
40 struct EntryFunctionState {
44 ExecutionMode getExecutionMode() const;
46 bool requiresFullRuntime() const { return RequiresFullRuntime; }
48 /// Get barrier to synchronize all threads in a block.
49 void syncCTAThreads(CodeGenFunction &CGF);
51 /// Helper for target directive initialization.
52 void emitKernelInit(CodeGenFunction &CGF, EntryFunctionState &EST,
55 /// Helper for target directive finalization.
56 void emitKernelDeinit(CodeGenFunction &CGF, EntryFunctionState &EST,
59 /// Helper for generic variables globalization prolog.
60 void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc,
61 bool WithSPMDCheck = false);
63 /// Helper for generic variables globalization epilog.
64 void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false);
67 // Base class overrides.
70 /// Creates offloading entry for the provided entry ID \a ID,
71 /// address \a Addr, size \a Size, and flags \a Flags.
72 void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr,
73 uint64_t Size, int32_t Flags,
74 llvm::GlobalValue::LinkageTypes Linkage) override;
76 /// Emit outlined function specialized for the Fork-Join
77 /// programming model for applicable target directives on the NVPTX device.
78 /// \param D Directive to emit.
79 /// \param ParentName Name of the function that encloses the target region.
80 /// \param OutlinedFn Outlined function value to be defined by this call.
81 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
82 /// \param IsOffloadEntry True if the outlined function is an offload entry.
83 /// An outlined function may not be an entry if, e.g. the if clause always
84 /// evaluates to false.
85 void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
86 llvm::Function *&OutlinedFn,
87 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
88 const RegionCodeGenTy &CodeGen);
90 /// Emit outlined function specialized for the Single Program
91 /// Multiple Data programming model for applicable target directives on the
93 /// \param D Directive to emit.
94 /// \param ParentName Name of the function that encloses the target region.
95 /// \param OutlinedFn Outlined function value to be defined by this call.
96 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
97 /// \param IsOffloadEntry True if the outlined function is an offload entry.
98 /// \param CodeGen Object containing the target statements.
99 /// An outlined function may not be an entry if, e.g. the if clause always
100 /// evaluates to false.
101 void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
102 llvm::Function *&OutlinedFn,
103 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
104 const RegionCodeGenTy &CodeGen);
106 /// Emit outlined function for 'target' directive on the NVPTX
108 /// \param D Directive to emit.
109 /// \param ParentName Name of the function that encloses the target region.
110 /// \param OutlinedFn Outlined function value to be defined by this call.
111 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
112 /// \param IsOffloadEntry True if the outlined function is an offload entry.
113 /// An outlined function may not be an entry if, e.g. the if clause always
114 /// evaluates to false.
115 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
116 StringRef ParentName,
117 llvm::Function *&OutlinedFn,
118 llvm::Constant *&OutlinedFnID,
120 const RegionCodeGenTy &CodeGen) override;
122 /// Emits code for parallel or serial call of the \a OutlinedFn with
123 /// variables captured in a record which address is stored in \a
125 /// This call is for the Non-SPMD Execution Mode.
126 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
127 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
128 /// \param CapturedVars A pointer to the record with the references to
129 /// variables used in \a OutlinedFn function.
130 /// \param IfCond Condition in the associated 'if' clause, if it was
131 /// specified, nullptr otherwise.
132 void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
133 llvm::Value *OutlinedFn,
134 ArrayRef<llvm::Value *> CapturedVars,
137 /// Emits code for parallel or serial call of the \a OutlinedFn with
138 /// variables captured in a record which address is stored in \a
140 /// This call is for a parallel directive within an SPMD target directive.
141 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
142 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
143 /// \param CapturedVars A pointer to the record with the references to
144 /// variables used in \a OutlinedFn function.
145 /// \param IfCond Condition in the associated 'if' clause, if it was
146 /// specified, nullptr otherwise.
148 void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
149 llvm::Function *OutlinedFn,
150 ArrayRef<llvm::Value *> CapturedVars,
154 /// Get the function name of an outlined region.
155 // The name can be customized depending on the target.
157 StringRef getOutlinedHelperName() const override {
158 return "__omp_outlined__";
161 /// Check if the default location must be constant.
162 /// Constant for NVPTX for better optimization.
163 bool isDefaultLocationConstant() const override { return true; }
165 /// Returns additional flags that can be stored in reserved_2 field of the
166 /// default location.
167 /// For NVPTX target contains data about SPMD/Non-SPMD execution mode +
168 /// Full/Lightweight runtime mode. Used for better optimization.
169 unsigned getDefaultLocationReserved2Flags() const override;
172 explicit CGOpenMPRuntimeGPU(CodeGenModule &CGM);
173 void clear() override;
175 /// Declare generalized virtual functions which need to be defined
176 /// by all specializations of OpenMPGPURuntime Targets like AMDGCN
179 /// Get the GPU warp size.
180 virtual llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) = 0;
182 /// Get the id of the current thread on the GPU.
183 virtual llvm::Value *getGPUThreadID(CodeGenFunction &CGF) = 0;
185 /// Get the maximum number of threads in a block of the GPU.
186 virtual llvm::Value *getGPUNumThreads(CodeGenFunction &CGF) = 0;
188 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
189 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
190 virtual void emitProcBindClause(CodeGenFunction &CGF,
191 llvm::omp::ProcBindKind ProcBind,
192 SourceLocation Loc) override;
194 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
195 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
197 /// \param NumThreads An integer value of threads.
198 virtual void emitNumThreadsClause(CodeGenFunction &CGF,
199 llvm::Value *NumThreads,
200 SourceLocation Loc) override;
202 /// This function ought to emit, in the general case, a call to
203 // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed
204 // as these numbers are obtained through the PTX grid and block configuration.
205 /// \param NumTeams An integer expression of teams.
206 /// \param ThreadLimit An integer expression of threads.
207 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
208 const Expr *ThreadLimit, SourceLocation Loc) override;
210 /// Emits inlined function for the specified OpenMP parallel
212 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
213 /// kmp_int32 BoundID, struct context_vars*).
214 /// \param D OpenMP directive.
215 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
216 /// \param InnermostKind Kind of innermost directive (for simple directives it
217 /// is a directive itself, for combined - its innermost directive).
218 /// \param CodeGen Code generation sequence for the \a D directive.
220 emitParallelOutlinedFunction(const OMPExecutableDirective &D,
221 const VarDecl *ThreadIDVar,
222 OpenMPDirectiveKind InnermostKind,
223 const RegionCodeGenTy &CodeGen) override;
225 /// Emits inlined function for the specified OpenMP teams
227 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
228 /// kmp_int32 BoundID, struct context_vars*).
229 /// \param D OpenMP directive.
230 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
231 /// \param InnermostKind Kind of innermost directive (for simple directives it
232 /// is a directive itself, for combined - its innermost directive).
233 /// \param CodeGen Code generation sequence for the \a D directive.
235 emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
236 const VarDecl *ThreadIDVar,
237 OpenMPDirectiveKind InnermostKind,
238 const RegionCodeGenTy &CodeGen) override;
240 /// Emits code for teams call of the \a OutlinedFn with
241 /// variables captured in a record which address is stored in \a
243 /// \param OutlinedFn Outlined function to be run by team masters. Type of
244 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
245 /// \param CapturedVars A pointer to the record with the references to
246 /// variables used in \a OutlinedFn function.
248 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
249 SourceLocation Loc, llvm::Function *OutlinedFn,
250 ArrayRef<llvm::Value *> CapturedVars) override;
252 /// Emits code for parallel or serial call of the \a OutlinedFn with
253 /// variables captured in a record which address is stored in \a
255 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
256 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
257 /// \param CapturedVars A pointer to the record with the references to
258 /// variables used in \a OutlinedFn function.
259 /// \param IfCond Condition in the associated 'if' clause, if it was
260 /// specified, nullptr otherwise.
261 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
262 llvm::Function *OutlinedFn,
263 ArrayRef<llvm::Value *> CapturedVars,
264 const Expr *IfCond) override;
266 /// Emit an implicit/explicit barrier for OpenMP threads.
267 /// \param Kind Directive for which this implicit barrier call must be
268 /// generated. Must be OMPD_barrier for explicit barrier generation.
269 /// \param EmitChecks true if need to emit checks for cancellation barriers.
270 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
271 /// runtime class decides which one to emit (simple or with cancellation
274 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
275 OpenMPDirectiveKind Kind, bool EmitChecks = true,
276 bool ForceSimpleCall = false) override;
278 /// Emits a critical region.
279 /// \param CriticalName Name of the critical region.
280 /// \param CriticalOpGen Generator for the statement associated with the given
282 /// \param Hint Value of the 'hint' clause (optional).
283 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
284 const RegionCodeGenTy &CriticalOpGen,
286 const Expr *Hint = nullptr) override;
288 /// Emit a code for reduction clause.
290 /// \param Privates List of private copies for original reduction arguments.
291 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
292 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
293 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
294 /// or 'operator binop(LHS, RHS)'.
295 /// \param Options List of options for reduction codegen:
296 /// WithNowait true if parent directive has also nowait clause, false
298 /// SimpleReduction Emit reduction operation only. Used for omp simd
299 /// directive on the host.
300 /// ReductionKind The kind of reduction to perform.
301 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
302 ArrayRef<const Expr *> Privates,
303 ArrayRef<const Expr *> LHSExprs,
304 ArrayRef<const Expr *> RHSExprs,
305 ArrayRef<const Expr *> ReductionOps,
306 ReductionOptionsTy Options) override;
308 /// Returns specified OpenMP runtime function for the current OpenMP
309 /// implementation. Specialized for the NVPTX device.
310 /// \param Function OpenMP runtime function.
311 /// \return Specified function.
312 llvm::FunctionCallee createNVPTXRuntimeFunction(unsigned Function);
314 /// Translates the native parameter of outlined function if this is required
316 /// \param FD Field decl from captured record for the parameter.
317 /// \param NativeParam Parameter itself.
318 const VarDecl *translateParameter(const FieldDecl *FD,
319 const VarDecl *NativeParam) const override;
321 /// Gets the address of the native argument basing on the address of the
322 /// target-specific parameter.
323 /// \param NativeParam Parameter itself.
324 /// \param TargetParam Corresponding target-specific parameter.
325 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
326 const VarDecl *TargetParam) const override;
328 /// Emits call of the outlined function with the provided arguments,
329 /// translating these arguments to correct target-specific arguments.
330 void emitOutlinedFunctionCall(
331 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
332 ArrayRef<llvm::Value *> Args = llvm::None) const override;
334 /// Emits OpenMP-specific function prolog.
335 /// Required for device constructs.
336 void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;
338 /// Gets the OpenMP-specific address of the local variable.
339 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
340 const VarDecl *VD) override;
342 /// Target codegen is specialized based on two data-sharing modes: CUDA, in
343 /// which the local variables are actually global threadlocal, and Generic, in
344 /// which the local variables are placed in global memory if they may escape
345 /// their declaration context.
346 enum DataSharingMode {
347 /// CUDA data sharing mode.
349 /// Generic data-sharing mode.
353 /// Cleans up references to the objects in finished function.
355 void functionFinished(CodeGenFunction &CGF) override;
357 /// Choose a default value for the dist_schedule clause.
358 void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
359 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
360 llvm::Value *&Chunk) const override;
362 /// Choose a default value for the schedule clause.
363 void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
364 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
365 const Expr *&ChunkExpr) const override;
367 /// Adjust some parameters for the target-based directives, like addresses of
368 /// the variables captured by reference in lambdas.
369 void adjustTargetSpecificDataForLambdas(
370 CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;
372 /// Perform check on requires decl to ensure that target architecture
373 /// supports unified addressing
374 void processRequiresDirective(const OMPRequiresDecl *D) override;
376 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
377 /// the predefined allocator and translates it into the corresponding address
379 bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override;
382 /// Track the execution mode when codegening directives within a target
383 /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the
384 /// target region and used by containing directives such as 'parallel'
385 /// to emit optimized code.
386 ExecutionMode CurrentExecutionMode = EM_Unknown;
388 /// Check if the full runtime is required (default - yes).
389 bool RequiresFullRuntime = true;
391 /// true if we're emitting the code for the target region and next parallel
392 /// region is L0 for sure.
393 bool IsInTargetMasterThreadRegion = false;
394 /// true if currently emitting code for target/teams/distribute region, false
396 bool IsInTTDRegion = false;
397 /// true if we're definitely in the parallel region.
398 bool IsInParallelRegion = false;
400 /// Map between an outlined function and its wrapper.
401 llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap;
403 /// Emit function which wraps the outline parallel region
404 /// and controls the parameters which are passed to this function.
405 /// The wrapper ensures that the outlined function is called
406 /// with the correct arguments when data is shared.
407 llvm::Function *createParallelDataSharingWrapper(
408 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);
410 /// The data for the single globalized variable.
411 struct MappedVarData {
412 /// Corresponding field in the global record.
413 llvm::Value *GlobalizedVal = nullptr;
414 /// Corresponding address.
415 Address PrivateAddr = Address::invalid();
417 /// The map of local variables to their addresses in the global memory.
418 using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
419 /// Set of the parameters passed by value escaping OpenMP context.
420 using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;
421 struct FunctionData {
422 DeclToAddrMapTy LocalVarData;
423 llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None;
424 EscapedParamsTy EscapedParameters;
425 llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
426 llvm::SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4>
427 EscapedVariableLengthDeclsAddrs;
428 llvm::Value *IsInSPMDModeFlag = nullptr;
429 std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
431 /// Maps the function to the list of the globalized variables with their
433 llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
434 llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr;
435 /// List of the records with the list of fields for the reductions across the
436 /// teams. Used to build the intermediate buffer for the fast teams
438 /// All the records are gathered into a union `union.type` is created.
439 llvm::SmallVector<const RecordDecl *, 4> TeamsReductions;
440 /// Shared pointer for the global memory in the global memory buffer used for
441 /// the given kernel.
442 llvm::GlobalVariable *KernelStaticGlobalized = nullptr;
443 /// Pair of the Non-SPMD team and all reductions variables in this team
445 std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>>
449 } // CodeGen namespace.
450 } // clang namespace.
452 #endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H