1 //===----- CGCUDANV.cpp - Interface to NVIDIA CUDA Runtime ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This provides a class for CUDA code generation targeting the NVIDIA CUDA
13 //===----------------------------------------------------------------------===//
15 #include "CGCUDARuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/Decl.h"
19 #include "clang/CodeGen/ConstantInitBuilder.h"
20 #include "llvm/IR/BasicBlock.h"
21 #include "llvm/IR/CallSite.h"
22 #include "llvm/IR/Constants.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/Format.h"
26 using namespace clang;
27 using namespace CodeGen;
30 constexpr unsigned CudaFatMagic = 0x466243b1;
31 constexpr unsigned HIPFatMagic = 0x48495046; // "HIPF"
33 class CGNVCUDARuntime : public CGCUDARuntime {
36 llvm::IntegerType *IntTy, *SizeTy;
38 llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy;
40 /// Convenience reference to LLVM Context
41 llvm::LLVMContext &Context;
42 /// Convenience reference to the current module
43 llvm::Module &TheModule;
44 /// Keeps track of kernel launch stubs emitted in this module
45 llvm::SmallVector<llvm::Function *, 16> EmittedKernels;
46 llvm::SmallVector<std::pair<llvm::GlobalVariable *, unsigned>, 16> DeviceVars;
47 /// Keeps track of variable containing handle of GPU binary. Populated by
48 /// ModuleCtorFunction() and used to create corresponding cleanup calls in
49 /// ModuleDtorFunction()
50 llvm::GlobalVariable *GpuBinaryHandle = nullptr;
51 /// Whether we generate relocatable device code.
52 bool RelocatableDeviceCode;
54 llvm::Constant *getSetupArgumentFn() const;
55 llvm::Constant *getLaunchFn() const;
57 llvm::FunctionType *getRegisterGlobalsFnTy() const;
58 llvm::FunctionType *getCallbackFnTy() const;
59 llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
60 std::string addPrefixToName(StringRef FuncName) const;
61 std::string addUnderscoredPrefixToName(StringRef FuncName) const;
63 /// Creates a function to register all kernel stubs generated in this module.
64 llvm::Function *makeRegisterGlobalsFn();
66 /// Helper function that generates a constant string and returns a pointer to
67 /// the start of the string. The result of this function can be used anywhere
68 /// where the C code specifies const char*.
69 llvm::Constant *makeConstantString(const std::string &Str,
70 const std::string &Name = "",
71 const std::string &SectionName = "",
72 unsigned Alignment = 0) {
73 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
74 llvm::ConstantInt::get(SizeTy, 0)};
75 auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
76 llvm::GlobalVariable *GV =
77 cast<llvm::GlobalVariable>(ConstStr.getPointer());
78 if (!SectionName.empty()) {
79 GV->setSection(SectionName);
80 // Mark the address as used which make sure that this section isn't
81 // merged and we will really have it in the object file.
82 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None);
85 GV->setAlignment(Alignment);
87 return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(),
88 ConstStr.getPointer(), Zeros);
91 /// Helper function that generates an empty dummy function returning void.
92 llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) {
93 assert(FnTy->getReturnType()->isVoidTy() &&
94 "Can only generate dummy functions returning void!");
95 llvm::Function *DummyFunc = llvm::Function::Create(
96 FnTy, llvm::GlobalValue::InternalLinkage, "dummy", &TheModule);
98 llvm::BasicBlock *DummyBlock =
99 llvm::BasicBlock::Create(Context, "", DummyFunc);
100 CGBuilderTy FuncBuilder(CGM, Context);
101 FuncBuilder.SetInsertPoint(DummyBlock);
102 FuncBuilder.CreateRetVoid();
107 void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args);
110 CGNVCUDARuntime(CodeGenModule &CGM);
112 void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override;
113 void registerDeviceVar(llvm::GlobalVariable &Var, unsigned Flags) override {
114 DeviceVars.push_back(std::make_pair(&Var, Flags));
117 /// Creates module constructor function
118 llvm::Function *makeModuleCtorFunction() override;
119 /// Creates module destructor function
120 llvm::Function *makeModuleDtorFunction() override;
125 std::string CGNVCUDARuntime::addPrefixToName(StringRef FuncName) const {
126 if (CGM.getLangOpts().HIP)
127 return ((Twine("hip") + Twine(FuncName)).str());
128 return ((Twine("cuda") + Twine(FuncName)).str());
131 CGNVCUDARuntime::addUnderscoredPrefixToName(StringRef FuncName) const {
132 if (CGM.getLangOpts().HIP)
133 return ((Twine("__hip") + Twine(FuncName)).str());
134 return ((Twine("__cuda") + Twine(FuncName)).str());
137 CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
138 : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()),
139 TheModule(CGM.getModule()),
140 RelocatableDeviceCode(CGM.getLangOpts().CUDARelocatableDeviceCode) {
141 CodeGen::CodeGenTypes &Types = CGM.getTypes();
142 ASTContext &Ctx = CGM.getContext();
148 CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy));
149 VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy));
150 VoidPtrPtrTy = VoidPtrTy->getPointerTo();
153 llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const {
154 // cudaError_t cudaSetupArgument(void *, size_t, size_t)
155 llvm::Type *Params[] = {VoidPtrTy, SizeTy, SizeTy};
156 return CGM.CreateRuntimeFunction(
157 llvm::FunctionType::get(IntTy, Params, false),
158 addPrefixToName("SetupArgument"));
161 llvm::Constant *CGNVCUDARuntime::getLaunchFn() const {
162 if (CGM.getLangOpts().HIP) {
163 // hipError_t hipLaunchByPtr(char *);
164 return CGM.CreateRuntimeFunction(
165 llvm::FunctionType::get(IntTy, CharPtrTy, false), "hipLaunchByPtr");
167 // cudaError_t cudaLaunch(char *);
168 return CGM.CreateRuntimeFunction(
169 llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch");
173 llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const {
174 return llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false);
177 llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const {
178 return llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
181 llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const {
182 auto CallbackFnTy = getCallbackFnTy();
183 auto RegisterGlobalsFnTy = getRegisterGlobalsFnTy();
184 llvm::Type *Params[] = {RegisterGlobalsFnTy->getPointerTo(), VoidPtrTy,
185 VoidPtrTy, CallbackFnTy->getPointerTo()};
186 return llvm::FunctionType::get(VoidTy, Params, false);
189 void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
190 FunctionArgList &Args) {
191 EmittedKernels.push_back(CGF.CurFn);
192 emitDeviceStubBody(CGF, Args);
195 void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF,
196 FunctionArgList &Args) {
197 // Emit a call to cudaSetupArgument for each arg in Args.
198 llvm::Constant *cudaSetupArgFn = getSetupArgumentFn();
199 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
200 CharUnits Offset = CharUnits::Zero();
201 for (const VarDecl *A : Args) {
202 CharUnits TyWidth, TyAlign;
203 std::tie(TyWidth, TyAlign) =
204 CGM.getContext().getTypeInfoInChars(A->getType());
205 Offset = Offset.alignTo(TyAlign);
206 llvm::Value *Args[] = {
207 CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(),
209 llvm::ConstantInt::get(SizeTy, TyWidth.getQuantity()),
210 llvm::ConstantInt::get(SizeTy, Offset.getQuantity()),
212 llvm::CallSite CS = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args);
213 llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0);
214 llvm::Value *CSZero = CGF.Builder.CreateICmpEQ(CS.getInstruction(), Zero);
215 llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next");
216 CGF.Builder.CreateCondBr(CSZero, NextBlock, EndBlock);
217 CGF.EmitBlock(NextBlock);
221 // Emit the call to cudaLaunch
222 llvm::Constant *cudaLaunchFn = getLaunchFn();
223 llvm::Value *Arg = CGF.Builder.CreatePointerCast(CGF.CurFn, CharPtrTy);
224 CGF.EmitRuntimeCallOrInvoke(cudaLaunchFn, Arg);
225 CGF.EmitBranch(EndBlock);
227 CGF.EmitBlock(EndBlock);
230 /// Creates a function that sets up state on the host side for CUDA objects that
231 /// have a presence on both the host and device sides. Specifically, registers
232 /// the host side of kernel functions and device global variables with the CUDA
235 /// void __cuda_register_globals(void** GpuBinaryHandle) {
236 /// __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...);
238 /// __cudaRegisterFunction(GpuBinaryHandle,KernelM,...);
239 /// __cudaRegisterVar(GpuBinaryHandle, GlobalVar0, ...);
241 /// __cudaRegisterVar(GpuBinaryHandle, GlobalVarN, ...);
244 llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
245 // No need to register anything
246 if (EmittedKernels.empty() && DeviceVars.empty())
249 llvm::Function *RegisterKernelsFunc = llvm::Function::Create(
250 getRegisterGlobalsFnTy(), llvm::GlobalValue::InternalLinkage,
251 addUnderscoredPrefixToName("_register_globals"), &TheModule);
252 llvm::BasicBlock *EntryBB =
253 llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc);
254 CGBuilderTy Builder(CGM, Context);
255 Builder.SetInsertPoint(EntryBB);
257 // void __cudaRegisterFunction(void **, const char *, char *, const char *,
258 // int, uint3*, uint3*, dim3*, dim3*, int*)
259 llvm::Type *RegisterFuncParams[] = {
260 VoidPtrPtrTy, CharPtrTy, CharPtrTy, CharPtrTy, IntTy,
261 VoidPtrTy, VoidPtrTy, VoidPtrTy, VoidPtrTy, IntTy->getPointerTo()};
262 llvm::Constant *RegisterFunc = CGM.CreateRuntimeFunction(
263 llvm::FunctionType::get(IntTy, RegisterFuncParams, false),
264 addUnderscoredPrefixToName("RegisterFunction"));
266 // Extract GpuBinaryHandle passed as the first argument passed to
267 // __cuda_register_globals() and generate __cudaRegisterFunction() call for
268 // each emitted kernel.
269 llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin();
270 for (llvm::Function *Kernel : EmittedKernels) {
271 llvm::Constant *KernelName = makeConstantString(Kernel->getName());
272 llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy);
273 llvm::Value *Args[] = {
274 &GpuBinaryHandlePtr, Builder.CreateBitCast(Kernel, VoidPtrTy),
275 KernelName, KernelName, llvm::ConstantInt::get(IntTy, -1), NullPtr,
276 NullPtr, NullPtr, NullPtr,
277 llvm::ConstantPointerNull::get(IntTy->getPointerTo())};
278 Builder.CreateCall(RegisterFunc, Args);
281 // void __cudaRegisterVar(void **, char *, char *, const char *,
282 // int, int, int, int)
283 llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy,
284 CharPtrTy, IntTy, IntTy,
286 llvm::Constant *RegisterVar = CGM.CreateRuntimeFunction(
287 llvm::FunctionType::get(IntTy, RegisterVarParams, false),
288 addUnderscoredPrefixToName("RegisterVar"));
289 for (auto &Pair : DeviceVars) {
290 llvm::GlobalVariable *Var = Pair.first;
291 unsigned Flags = Pair.second;
292 llvm::Constant *VarName = makeConstantString(Var->getName());
294 CGM.getDataLayout().getTypeAllocSize(Var->getValueType());
295 llvm::Value *Args[] = {
297 Builder.CreateBitCast(Var, VoidPtrTy),
300 llvm::ConstantInt::get(IntTy, (Flags & ExternDeviceVar) ? 1 : 0),
301 llvm::ConstantInt::get(IntTy, VarSize),
302 llvm::ConstantInt::get(IntTy, (Flags & ConstantDeviceVar) ? 1 : 0),
303 llvm::ConstantInt::get(IntTy, 0)};
304 Builder.CreateCall(RegisterVar, Args);
307 Builder.CreateRetVoid();
308 return RegisterKernelsFunc;
311 /// Creates a global constructor function for the module:
315 /// void __cuda_module_ctor(void*) {
316 /// Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
317 /// __cuda_register_globals(Handle);
323 /// void __hip_module_ctor(void*) {
324 /// if (__hip_gpubin_handle == 0) {
325 /// __hip_gpubin_handle = __hipRegisterFatBinary(GpuBinaryBlob);
326 /// __hip_register_globals(__hip_gpubin_handle);
330 llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
331 bool IsHIP = CGM.getLangOpts().HIP;
332 // No need to generate ctors/dtors if there is no GPU binary.
333 StringRef CudaGpuBinaryFileName = CGM.getCodeGenOpts().CudaGpuBinaryFileName;
334 if (CudaGpuBinaryFileName.empty() && !IsHIP)
337 // void __{cuda|hip}_register_globals(void* handle);
338 llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn();
339 // We always need a function to pass in as callback. Create a dummy
340 // implementation if we don't need to register anything.
341 if (RelocatableDeviceCode && !RegisterGlobalsFunc)
342 RegisterGlobalsFunc = makeDummyFunction(getRegisterGlobalsFnTy());
344 // void ** __{cuda|hip}RegisterFatBinary(void *);
345 llvm::Constant *RegisterFatbinFunc = CGM.CreateRuntimeFunction(
346 llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false),
347 addUnderscoredPrefixToName("RegisterFatBinary"));
348 // struct { int magic, int version, void * gpu_binary, void * dont_care };
349 llvm::StructType *FatbinWrapperTy =
350 llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy);
352 // Register GPU binary with the CUDA runtime, store returned handle in a
353 // global variable and save a reference in GpuBinaryHandle to be cleaned up
354 // in destructor on exit. Then associate all known kernels with the GPU binary
355 // handle so CUDA runtime can figure out what to call on the GPU side.
356 std::unique_ptr<llvm::MemoryBuffer> CudaGpuBinary;
358 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> CudaGpuBinaryOrErr =
359 llvm::MemoryBuffer::getFileOrSTDIN(CudaGpuBinaryFileName);
360 if (std::error_code EC = CudaGpuBinaryOrErr.getError()) {
361 CGM.getDiags().Report(diag::err_cannot_open_file)
362 << CudaGpuBinaryFileName << EC.message();
365 CudaGpuBinary = std::move(CudaGpuBinaryOrErr.get());
368 llvm::Function *ModuleCtorFunc = llvm::Function::Create(
369 llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
370 llvm::GlobalValue::InternalLinkage,
371 addUnderscoredPrefixToName("_module_ctor"), &TheModule);
372 llvm::BasicBlock *CtorEntryBB =
373 llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc);
374 CGBuilderTy CtorBuilder(CGM, Context);
376 CtorBuilder.SetInsertPoint(CtorEntryBB);
378 const char *FatbinConstantName;
379 const char *FatbinSectionName;
380 const char *ModuleIDSectionName;
381 StringRef ModuleIDPrefix;
382 llvm::Constant *FatBinStr;
385 FatbinConstantName = ".hip_fatbin";
386 FatbinSectionName = ".hipFatBinSegment";
388 ModuleIDSectionName = "__hip_module_id";
389 ModuleIDPrefix = "__hip_";
391 // For HIP, create an external symbol __hip_fatbin in section .hip_fatbin.
392 // The external symbol is supposed to contain the fat binary but will be
393 // populated somewhere else, e.g. by lld through link script.
394 FatBinStr = new llvm::GlobalVariable(
395 CGM.getModule(), CGM.Int8Ty,
396 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr,
397 "__hip_fatbin", nullptr,
398 llvm::GlobalVariable::NotThreadLocal);
399 cast<llvm::GlobalVariable>(FatBinStr)->setSection(FatbinConstantName);
401 FatMagic = HIPFatMagic;
403 if (RelocatableDeviceCode)
404 FatbinConstantName = CGM.getTriple().isMacOSX()
405 ? "__NV_CUDA,__nv_relfatbin"
409 CGM.getTriple().isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin";
410 // NVIDIA's cuobjdump looks for fatbins in this section.
412 CGM.getTriple().isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment";
414 ModuleIDSectionName = CGM.getTriple().isMacOSX()
415 ? "__NV_CUDA,__nv_module_id"
417 ModuleIDPrefix = "__nv_";
419 // For CUDA, create a string literal containing the fat binary loaded from
421 FatBinStr = makeConstantString(CudaGpuBinary->getBuffer(), "",
422 FatbinConstantName, 8);
423 FatMagic = CudaFatMagic;
426 // Create initialized wrapper structure that points to the loaded GPU binary
427 ConstantInitBuilder Builder(CGM);
428 auto Values = Builder.beginStruct(FatbinWrapperTy);
429 // Fatbin wrapper magic.
430 Values.addInt(IntTy, FatMagic);
432 Values.addInt(IntTy, 1);
434 Values.add(FatBinStr);
435 // Unused in fatbin v1.
436 Values.add(llvm::ConstantPointerNull::get(VoidPtrTy));
437 llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal(
438 addUnderscoredPrefixToName("_fatbin_wrapper"), CGM.getPointerAlign(),
440 FatbinWrapper->setSection(FatbinSectionName);
442 // There is only one HIP fat binary per linked module, however there are
443 // multiple constructor functions. Make sure the fat binary is registered
444 // only once. The constructor functions are executed by the dynamic loader
445 // before the program gains control. The dynamic loader cannot execute the
446 // constructor functions concurrently since doing that would not guarantee
447 // thread safety of the loaded program. Therefore we can assume sequential
448 // execution of constructor functions here.
450 llvm::BasicBlock *IfBlock =
451 llvm::BasicBlock::Create(Context, "if", ModuleCtorFunc);
452 llvm::BasicBlock *ExitBlock =
453 llvm::BasicBlock::Create(Context, "exit", ModuleCtorFunc);
454 // The name, size, and initialization pattern of this variable is part
456 GpuBinaryHandle = new llvm::GlobalVariable(
457 TheModule, VoidPtrPtrTy, /*isConstant=*/false,
458 llvm::GlobalValue::LinkOnceAnyLinkage,
459 /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy),
460 "__hip_gpubin_handle");
461 GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getQuantity());
462 Address GpuBinaryAddr(
464 CharUnits::fromQuantity(GpuBinaryHandle->getAlignment()));
466 auto HandleValue = CtorBuilder.CreateLoad(GpuBinaryAddr);
467 llvm::Constant *Zero =
468 llvm::Constant::getNullValue(HandleValue->getType());
469 llvm::Value *EQZero = CtorBuilder.CreateICmpEQ(HandleValue, Zero);
470 CtorBuilder.CreateCondBr(EQZero, IfBlock, ExitBlock);
473 CtorBuilder.SetInsertPoint(IfBlock);
474 // GpuBinaryHandle = __hipRegisterFatBinary(&FatbinWrapper);
475 llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(
477 CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy));
478 CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryAddr);
479 CtorBuilder.CreateBr(ExitBlock);
482 CtorBuilder.SetInsertPoint(ExitBlock);
483 // Call __hip_register_globals(GpuBinaryHandle);
484 if (RegisterGlobalsFunc) {
485 auto HandleValue = CtorBuilder.CreateLoad(GpuBinaryAddr);
486 CtorBuilder.CreateCall(RegisterGlobalsFunc, HandleValue);
489 } else if (!RelocatableDeviceCode) {
490 // Register binary with CUDA runtime. This is substantially different in
491 // default mode vs. separate compilation!
492 // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper);
493 llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(
495 CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy));
496 GpuBinaryHandle = new llvm::GlobalVariable(
497 TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage,
498 llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle");
499 GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getQuantity());
500 CtorBuilder.CreateAlignedStore(RegisterFatbinCall, GpuBinaryHandle,
501 CGM.getPointerAlign());
503 // Call __cuda_register_globals(GpuBinaryHandle);
504 if (RegisterGlobalsFunc)
505 CtorBuilder.CreateCall(RegisterGlobalsFunc, RegisterFatbinCall);
507 // Generate a unique module ID.
508 SmallString<64> ModuleID;
509 llvm::raw_svector_ostream OS(ModuleID);
510 OS << ModuleIDPrefix << llvm::format("%x", FatbinWrapper->getGUID());
511 llvm::Constant *ModuleIDConstant =
512 makeConstantString(ModuleID.str(), "", ModuleIDSectionName, 32);
514 // Create an alias for the FatbinWrapper that nvcc will look for.
515 llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage,
516 Twine("__fatbinwrap") + ModuleID, FatbinWrapper);
518 // void __cudaRegisterLinkedBinary%ModuleID%(void (*)(void *), void *,
519 // void *, void (*)(void **))
520 SmallString<128> RegisterLinkedBinaryName("__cudaRegisterLinkedBinary");
521 RegisterLinkedBinaryName += ModuleID;
522 llvm::Constant *RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction(
523 getRegisterLinkedBinaryFnTy(), RegisterLinkedBinaryName);
525 assert(RegisterGlobalsFunc && "Expecting at least dummy function!");
526 llvm::Value *Args[] = {RegisterGlobalsFunc,
527 CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy),
529 makeDummyFunction(getCallbackFnTy())};
530 CtorBuilder.CreateCall(RegisterLinkedBinaryFunc, Args);
533 // Create destructor and register it with atexit() the way NVCC does it. Doing
534 // it during regular destructor phase worked in CUDA before 9.2 but results in
535 // double-free in 9.2.
536 if (llvm::Function *CleanupFn = makeModuleDtorFunction()) {
537 // extern "C" int atexit(void (*f)(void));
538 llvm::FunctionType *AtExitTy =
539 llvm::FunctionType::get(IntTy, CleanupFn->getType(), false);
540 llvm::Constant *AtExitFunc =
541 CGM.CreateRuntimeFunction(AtExitTy, "atexit", llvm::AttributeList(),
543 CtorBuilder.CreateCall(AtExitFunc, CleanupFn);
546 CtorBuilder.CreateRetVoid();
547 return ModuleCtorFunc;
550 /// Creates a global destructor function that unregisters the GPU code blob
551 /// registered by constructor.
555 /// void __cuda_module_dtor(void*) {
556 /// __cudaUnregisterFatBinary(Handle);
562 /// void __hip_module_dtor(void*) {
563 /// if (__hip_gpubin_handle) {
564 /// __hipUnregisterFatBinary(__hip_gpubin_handle);
565 /// __hip_gpubin_handle = 0;
569 llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
570 // No need for destructor if we don't have a handle to unregister.
571 if (!GpuBinaryHandle)
574 // void __cudaUnregisterFatBinary(void ** handle);
575 llvm::Constant *UnregisterFatbinFunc = CGM.CreateRuntimeFunction(
576 llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false),
577 addUnderscoredPrefixToName("UnregisterFatBinary"));
579 llvm::Function *ModuleDtorFunc = llvm::Function::Create(
580 llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
581 llvm::GlobalValue::InternalLinkage,
582 addUnderscoredPrefixToName("_module_dtor"), &TheModule);
584 llvm::BasicBlock *DtorEntryBB =
585 llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc);
586 CGBuilderTy DtorBuilder(CGM, Context);
587 DtorBuilder.SetInsertPoint(DtorEntryBB);
589 Address GpuBinaryAddr(GpuBinaryHandle, CharUnits::fromQuantity(
590 GpuBinaryHandle->getAlignment()));
591 auto HandleValue = DtorBuilder.CreateLoad(GpuBinaryAddr);
592 // There is only one HIP fat binary per linked module, however there are
593 // multiple destructor functions. Make sure the fat binary is unregistered
595 if (CGM.getLangOpts().HIP) {
596 llvm::BasicBlock *IfBlock =
597 llvm::BasicBlock::Create(Context, "if", ModuleDtorFunc);
598 llvm::BasicBlock *ExitBlock =
599 llvm::BasicBlock::Create(Context, "exit", ModuleDtorFunc);
600 llvm::Constant *Zero = llvm::Constant::getNullValue(HandleValue->getType());
601 llvm::Value *NEZero = DtorBuilder.CreateICmpNE(HandleValue, Zero);
602 DtorBuilder.CreateCondBr(NEZero, IfBlock, ExitBlock);
604 DtorBuilder.SetInsertPoint(IfBlock);
605 DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue);
606 DtorBuilder.CreateStore(Zero, GpuBinaryAddr);
607 DtorBuilder.CreateBr(ExitBlock);
609 DtorBuilder.SetInsertPoint(ExitBlock);
611 DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue);
613 DtorBuilder.CreateRetVoid();
614 return ModuleDtorFunc;
617 CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) {
618 return new CGNVCUDARuntime(CGM);