1 //===----- CGOpenMPRuntimeNVPTX.h - Interface to OpenMP NVPTX Runtimes ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This provides a class for OpenMP runtime code generation specialized to NVPTX
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
15 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H
17 #include "CGOpenMPRuntime.h"
18 #include "CodeGenFunction.h"
19 #include "clang/AST/StmtOpenMP.h"
24 class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntime {
26 /// Defines the execution mode.
28 /// SPMD execution mode (all threads are worker threads).
30 /// Non-SPMD execution mode (1 master thread, others are workers).
32 /// Unknown execution mode (orphaned directive).
36 /// Parallel outlined function work for workers to execute.
37 llvm::SmallVector<llvm::Function *, 16> Work;
39 struct EntryFunctionState {
40 llvm::BasicBlock *ExitBB = nullptr;
43 class WorkerFunctionState {
45 llvm::Function *WorkerFn;
46 const CGFunctionInfo &CGFI;
49 WorkerFunctionState(CodeGenModule &CGM, SourceLocation Loc);
52 void createWorkerFunction(CodeGenModule &CGM);
55 ExecutionMode getExecutionMode() const;
57 bool requiresFullRuntime() const { return RequiresFullRuntime; }
59 /// Get barrier to synchronize all threads in a block.
60 void syncCTAThreads(CodeGenFunction &CGF);
62 /// Emit the worker function for the current target region.
63 void emitWorkerFunction(WorkerFunctionState &WST);
65 /// Helper for worker function. Emit body of worker loop.
66 void emitWorkerLoop(CodeGenFunction &CGF, WorkerFunctionState &WST);
68 /// Helper for non-SPMD target entry function. Guide the master and
69 /// worker threads to their respective locations.
70 void emitNonSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST,
71 WorkerFunctionState &WST);
73 /// Signal termination of OMP execution for non-SPMD target entry
75 void emitNonSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST);
77 /// Helper for generic variables globalization prolog.
78 void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc,
79 bool WithSPMDCheck = false);
81 /// Helper for generic variables globalization epilog.
82 void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false);
84 /// Helper for SPMD mode target directive's entry function.
85 void emitSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST,
86 const OMPExecutableDirective &D);
88 /// Signal termination of SPMD mode execution.
89 void emitSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST);
92 // Base class overrides.
95 /// Creates offloading entry for the provided entry ID \a ID,
96 /// address \a Addr, size \a Size, and flags \a Flags.
97 void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr,
98 uint64_t Size, int32_t Flags,
99 llvm::GlobalValue::LinkageTypes Linkage) override;
101 /// Emit outlined function specialized for the Fork-Join
102 /// programming model for applicable target directives on the NVPTX device.
103 /// \param D Directive to emit.
104 /// \param ParentName Name of the function that encloses the target region.
105 /// \param OutlinedFn Outlined function value to be defined by this call.
106 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
107 /// \param IsOffloadEntry True if the outlined function is an offload entry.
108 /// An outlined function may not be an entry if, e.g. the if clause always
109 /// evaluates to false.
110 void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
111 llvm::Function *&OutlinedFn,
112 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
113 const RegionCodeGenTy &CodeGen);
115 /// Emit outlined function specialized for the Single Program
116 /// Multiple Data programming model for applicable target directives on the
118 /// \param D Directive to emit.
119 /// \param ParentName Name of the function that encloses the target region.
120 /// \param OutlinedFn Outlined function value to be defined by this call.
121 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
122 /// \param IsOffloadEntry True if the outlined function is an offload entry.
123 /// \param CodeGen Object containing the target statements.
124 /// An outlined function may not be an entry if, e.g. the if clause always
125 /// evaluates to false.
126 void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
127 llvm::Function *&OutlinedFn,
128 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
129 const RegionCodeGenTy &CodeGen);
131 /// Emit outlined function for 'target' directive on the NVPTX
133 /// \param D Directive to emit.
134 /// \param ParentName Name of the function that encloses the target region.
135 /// \param OutlinedFn Outlined function value to be defined by this call.
136 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
137 /// \param IsOffloadEntry True if the outlined function is an offload entry.
138 /// An outlined function may not be an entry if, e.g. the if clause always
139 /// evaluates to false.
140 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
141 StringRef ParentName,
142 llvm::Function *&OutlinedFn,
143 llvm::Constant *&OutlinedFnID,
145 const RegionCodeGenTy &CodeGen) override;
147 /// Emits code for parallel or serial call of the \a OutlinedFn with
148 /// variables captured in a record which address is stored in \a
150 /// This call is for the Non-SPMD Execution Mode.
151 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
152 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
153 /// \param CapturedVars A pointer to the record with the references to
154 /// variables used in \a OutlinedFn function.
155 /// \param IfCond Condition in the associated 'if' clause, if it was
156 /// specified, nullptr otherwise.
157 void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
158 llvm::Value *OutlinedFn,
159 ArrayRef<llvm::Value *> CapturedVars,
162 /// Emits code for parallel or serial call of the \a OutlinedFn with
163 /// variables captured in a record which address is stored in \a
165 /// This call is for a parallel directive within an SPMD target directive.
166 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
167 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
168 /// \param CapturedVars A pointer to the record with the references to
169 /// variables used in \a OutlinedFn function.
170 /// \param IfCond Condition in the associated 'if' clause, if it was
171 /// specified, nullptr otherwise.
173 void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
174 llvm::Function *OutlinedFn,
175 ArrayRef<llvm::Value *> CapturedVars,
179 /// Get the function name of an outlined region.
180 // The name can be customized depending on the target.
182 StringRef getOutlinedHelperName() const override {
183 return "__omp_outlined__";
186 /// Check if the default location must be constant.
187 /// Constant for NVPTX for better optimization.
188 bool isDefaultLocationConstant() const override { return true; }
190 /// Returns additional flags that can be stored in reserved_2 field of the
191 /// default location.
192 /// For NVPTX target contains data about SPMD/Non-SPMD execution mode +
193 /// Full/Lightweight runtime mode. Used for better optimization.
194 unsigned getDefaultLocationReserved2Flags() const override;
196 /// Tries to emit declare variant function for \p OldGD from \p NewGD.
197 /// \param OrigAddr LLVM IR value for \p OldGD.
198 /// \param IsForDefinition true, if requested emission for the definition of
200 /// \returns true, was able to emit a definition function for \p OldGD, which
201 /// points to \p NewGD.
202 /// NVPTX backend does not support global aliases, so just use the function,
203 /// emitted for \p NewGD instead of \p OldGD.
204 bool tryEmitDeclareVariant(const GlobalDecl &NewGD, const GlobalDecl &OldGD,
205 llvm::GlobalValue *OrigAddr,
206 bool IsForDefinition) override;
209 explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM);
210 void clear() override;
212 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
213 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
214 virtual void emitProcBindClause(CodeGenFunction &CGF,
215 llvm::omp::ProcBindKind ProcBind,
216 SourceLocation Loc) override;
218 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
219 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
221 /// \param NumThreads An integer value of threads.
222 virtual void emitNumThreadsClause(CodeGenFunction &CGF,
223 llvm::Value *NumThreads,
224 SourceLocation Loc) override;
226 /// This function ought to emit, in the general case, a call to
227 // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed
228 // as these numbers are obtained through the PTX grid and block configuration.
229 /// \param NumTeams An integer expression of teams.
230 /// \param ThreadLimit An integer expression of threads.
231 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
232 const Expr *ThreadLimit, SourceLocation Loc) override;
234 /// Emits inlined function for the specified OpenMP parallel
236 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
237 /// kmp_int32 BoundID, struct context_vars*).
238 /// \param D OpenMP directive.
239 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
240 /// \param InnermostKind Kind of innermost directive (for simple directives it
241 /// is a directive itself, for combined - its innermost directive).
242 /// \param CodeGen Code generation sequence for the \a D directive.
244 emitParallelOutlinedFunction(const OMPExecutableDirective &D,
245 const VarDecl *ThreadIDVar,
246 OpenMPDirectiveKind InnermostKind,
247 const RegionCodeGenTy &CodeGen) override;
249 /// Emits inlined function for the specified OpenMP teams
251 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
252 /// kmp_int32 BoundID, struct context_vars*).
253 /// \param D OpenMP directive.
254 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
255 /// \param InnermostKind Kind of innermost directive (for simple directives it
256 /// is a directive itself, for combined - its innermost directive).
257 /// \param CodeGen Code generation sequence for the \a D directive.
259 emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
260 const VarDecl *ThreadIDVar,
261 OpenMPDirectiveKind InnermostKind,
262 const RegionCodeGenTy &CodeGen) override;
264 /// Emits code for teams call of the \a OutlinedFn with
265 /// variables captured in a record which address is stored in \a
267 /// \param OutlinedFn Outlined function to be run by team masters. Type of
268 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
269 /// \param CapturedVars A pointer to the record with the references to
270 /// variables used in \a OutlinedFn function.
272 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
273 SourceLocation Loc, llvm::Function *OutlinedFn,
274 ArrayRef<llvm::Value *> CapturedVars) override;
276 /// Emits code for parallel or serial call of the \a OutlinedFn with
277 /// variables captured in a record which address is stored in \a
279 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
280 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
281 /// \param CapturedVars A pointer to the record with the references to
282 /// variables used in \a OutlinedFn function.
283 /// \param IfCond Condition in the associated 'if' clause, if it was
284 /// specified, nullptr otherwise.
285 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
286 llvm::Function *OutlinedFn,
287 ArrayRef<llvm::Value *> CapturedVars,
288 const Expr *IfCond) override;
290 /// Emit an implicit/explicit barrier for OpenMP threads.
291 /// \param Kind Directive for which this implicit barrier call must be
292 /// generated. Must be OMPD_barrier for explicit barrier generation.
293 /// \param EmitChecks true if need to emit checks for cancellation barriers.
294 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
295 /// runtime class decides which one to emit (simple or with cancellation
298 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
299 OpenMPDirectiveKind Kind, bool EmitChecks = true,
300 bool ForceSimpleCall = false) override;
302 /// Emits a critical region.
303 /// \param CriticalName Name of the critical region.
304 /// \param CriticalOpGen Generator for the statement associated with the given
306 /// \param Hint Value of the 'hint' clause (optional).
307 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
308 const RegionCodeGenTy &CriticalOpGen,
310 const Expr *Hint = nullptr) override;
312 /// Emit a code for reduction clause.
314 /// \param Privates List of private copies for original reduction arguments.
315 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
316 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
317 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
318 /// or 'operator binop(LHS, RHS)'.
319 /// \param Options List of options for reduction codegen:
320 /// WithNowait true if parent directive has also nowait clause, false
322 /// SimpleReduction Emit reduction operation only. Used for omp simd
323 /// directive on the host.
324 /// ReductionKind The kind of reduction to perform.
325 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
326 ArrayRef<const Expr *> Privates,
327 ArrayRef<const Expr *> LHSExprs,
328 ArrayRef<const Expr *> RHSExprs,
329 ArrayRef<const Expr *> ReductionOps,
330 ReductionOptionsTy Options) override;
332 /// Returns specified OpenMP runtime function for the current OpenMP
333 /// implementation. Specialized for the NVPTX device.
334 /// \param Function OpenMP runtime function.
335 /// \return Specified function.
336 llvm::FunctionCallee createNVPTXRuntimeFunction(unsigned Function);
338 /// Translates the native parameter of outlined function if this is required
340 /// \param FD Field decl from captured record for the parameter.
341 /// \param NativeParam Parameter itself.
342 const VarDecl *translateParameter(const FieldDecl *FD,
343 const VarDecl *NativeParam) const override;
345 /// Gets the address of the native argument basing on the address of the
346 /// target-specific parameter.
347 /// \param NativeParam Parameter itself.
348 /// \param TargetParam Corresponding target-specific parameter.
349 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
350 const VarDecl *TargetParam) const override;
352 /// Emits call of the outlined function with the provided arguments,
353 /// translating these arguments to correct target-specific arguments.
354 void emitOutlinedFunctionCall(
355 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
356 ArrayRef<llvm::Value *> Args = llvm::None) const override;
358 /// Emits OpenMP-specific function prolog.
359 /// Required for device constructs.
360 void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;
362 /// Gets the OpenMP-specific address of the local variable.
363 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
364 const VarDecl *VD) override;
366 /// Target codegen is specialized based on two data-sharing modes: CUDA, in
367 /// which the local variables are actually global threadlocal, and Generic, in
368 /// which the local variables are placed in global memory if they may escape
369 /// their declaration context.
370 enum DataSharingMode {
371 /// CUDA data sharing mode.
373 /// Generic data-sharing mode.
377 /// Cleans up references to the objects in finished function.
379 void functionFinished(CodeGenFunction &CGF) override;
381 /// Choose a default value for the dist_schedule clause.
382 void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
383 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
384 llvm::Value *&Chunk) const override;
386 /// Choose a default value for the schedule clause.
387 void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
388 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
389 const Expr *&ChunkExpr) const override;
391 /// Adjust some parameters for the target-based directives, like addresses of
392 /// the variables captured by reference in lambdas.
393 void adjustTargetSpecificDataForLambdas(
394 CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;
396 /// Perform check on requires decl to ensure that target architecture
397 /// supports unified addressing
398 void checkArchForUnifiedAddressing(const OMPRequiresDecl *D) override;
400 /// Returns default address space for the constant firstprivates, __constant__
401 /// address space by default.
402 unsigned getDefaultFirstprivateAddressSpace() const override;
404 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
405 /// the predefined allocator and translates it into the corresponding address
407 bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override;
410 /// Track the execution mode when codegening directives within a target
411 /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the
412 /// target region and used by containing directives such as 'parallel'
413 /// to emit optimized code.
414 ExecutionMode CurrentExecutionMode = EM_Unknown;
416 /// Check if the full runtime is required (default - yes).
417 bool RequiresFullRuntime = true;
419 /// true if we're emitting the code for the target region and next parallel
420 /// region is L0 for sure.
421 bool IsInTargetMasterThreadRegion = false;
422 /// true if currently emitting code for target/teams/distribute region, false
424 bool IsInTTDRegion = false;
425 /// true if we're definitely in the parallel region.
426 bool IsInParallelRegion = false;
428 /// Map between an outlined function and its wrapper.
429 llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap;
431 /// Emit function which wraps the outline parallel region
432 /// and controls the parameters which are passed to this function.
433 /// The wrapper ensures that the outlined function is called
434 /// with the correct arguments when data is shared.
435 llvm::Function *createParallelDataSharingWrapper(
436 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);
438 /// The data for the single globalized variable.
439 struct MappedVarData {
440 /// Corresponding field in the global record.
441 const FieldDecl *FD = nullptr;
442 /// Corresponding address.
443 Address PrivateAddr = Address::invalid();
444 /// true, if only one element is required (for latprivates in SPMD mode),
445 /// false, if need to create based on the warp-size.
446 bool IsOnePerTeam = false;
447 MappedVarData() = delete;
448 MappedVarData(const FieldDecl *FD, bool IsOnePerTeam = false)
449 : FD(FD), IsOnePerTeam(IsOnePerTeam) {}
451 /// The map of local variables to their addresses in the global memory.
452 using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
453 /// Set of the parameters passed by value escaping OpenMP context.
454 using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;
455 struct FunctionData {
456 DeclToAddrMapTy LocalVarData;
457 llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None;
458 EscapedParamsTy EscapedParameters;
459 llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
460 llvm::SmallVector<llvm::Value *, 4> EscapedVariableLengthDeclsAddrs;
461 const RecordDecl *GlobalRecord = nullptr;
462 llvm::Optional<const RecordDecl *> SecondaryGlobalRecord = llvm::None;
463 llvm::Value *GlobalRecordAddr = nullptr;
464 llvm::Value *IsInSPMDModeFlag = nullptr;
465 std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
467 /// Maps the function to the list of the globalized variables with their
469 llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
470 /// List of records for the globalized variables in target/teams/distribute
471 /// contexts. Inner records are going to be joined into the single record,
472 /// while those resulting records are going to be joined into the single
473 /// union. This resulting union (one per CU) is the entry point for the static
474 /// memory management runtime functions.
475 struct GlobalPtrSizeRecsTy {
476 llvm::GlobalVariable *UseSharedMemory = nullptr;
477 llvm::GlobalVariable *RecSize = nullptr;
478 llvm::GlobalVariable *Buffer = nullptr;
480 llvm::SmallVector<const RecordDecl *, 2> Records;
481 unsigned RegionCounter = 0;
483 llvm::SmallVector<GlobalPtrSizeRecsTy, 8> GlobalizedRecords;
484 llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr;
485 /// List of the records with the list of fields for the reductions across the
486 /// teams. Used to build the intermediate buffer for the fast teams
488 /// All the records are gathered into a union `union.type` is created.
489 llvm::SmallVector<const RecordDecl *, 4> TeamsReductions;
490 /// Shared pointer for the global memory in the global memory buffer used for
491 /// the given kernel.
492 llvm::GlobalVariable *KernelStaticGlobalized = nullptr;
493 /// Pair of the Non-SPMD team and all reductions variables in this team
495 std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>>
499 } // CodeGen namespace.
500 } // clang namespace.
502 #endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H