contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp

   1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
  10 #include "llvm/ADT/STLExtras.h"
  11 #include "llvm/ADT/Triple.h"
  12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
  13 #include "llvm/IR/CallSite.h"
  14 #include "llvm/IR/IRBuilder.h"
  15 #include "llvm/Support/Format.h"
  16 #include "llvm/Transforms/Utils/Cloning.h"
  17 #include <sstream>
  18
  19 using namespace llvm;
  20 using namespace llvm::orc;
  21
  22 namespace {
  23
  24 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
  25 public:
  26   using CompileFunction = JITCompileCallbackManager::CompileFunction;
  27
  28   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
  29                                      CompileFunction Compile, VModuleKey K)
  30       : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}),
  31                             std::move(K)),
  32         Name(std::move(Name)), Compile(std::move(Compile)) {}
  33
  34   StringRef getName() const override { return "<Compile Callbacks>"; }
  35
  36 private:
  37   void materialize(MaterializationResponsibility R) override {
  38     SymbolMap Result;
  39     Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
  40     R.notifyResolved(Result);
  41     R.notifyEmitted();
  42   }
  43
  44   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
  45     llvm_unreachable("Discard should never occur on a LMU?");
  46   }
  47
  48   SymbolStringPtr Name;
  49   CompileFunction Compile;
  50 };
  51
  52 } // namespace
  53
  54 namespace llvm {
  55 namespace orc {
  56
  57 void IndirectStubsManager::anchor() {}
  58 void TrampolinePool::anchor() {}
  59
  60 Expected<JITTargetAddress>
  61 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
  62   if (auto TrampolineAddr = TP->getTrampoline()) {
  63     auto CallbackName =
  64         ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
  65
  66     std::lock_guard<std::mutex> Lock(CCMgrMutex);
  67     AddrToSymbol[*TrampolineAddr] = CallbackName;
  68     cantFail(CallbacksJD.define(
  69         llvm::make_unique<CompileCallbackMaterializationUnit>(
  70             std::move(CallbackName), std::move(Compile),
  71             ES.allocateVModule())));
  72     return *TrampolineAddr;
  73   } else
  74     return TrampolineAddr.takeError();
  75 }
  76
  77 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
  78     JITTargetAddress TrampolineAddr) {
  79   SymbolStringPtr Name;
  80
  81   {
  82     std::unique_lock<std::mutex> Lock(CCMgrMutex);
  83     auto I = AddrToSymbol.find(TrampolineAddr);
  84
  85     // If this address is not associated with a compile callback then report an
  86     // error to the execution session and return ErrorHandlerAddress to the
  87     // callee.
  88     if (I == AddrToSymbol.end()) {
  89       Lock.unlock();
  90       std::string ErrMsg;
  91       {
  92         raw_string_ostream ErrMsgStream(ErrMsg);
  93         ErrMsgStream << "No compile callback for trampoline at "
  94                      << format("0x%016" PRIx64, TrampolineAddr);
  95       }
  96       ES.reportError(
  97           make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
  98       return ErrorHandlerAddress;
  99     } else
 100       Name = I->second;
 101   }
 102
 103   if (auto Sym = ES.lookup(JITDylibSearchList({{&CallbacksJD, true}}), Name))
 104     return Sym->getAddress();
 105   else {
 106     llvm::dbgs() << "Didn't find callback.\n";
 107     // If anything goes wrong materializing Sym then report it to the session
 108     // and return the ErrorHandlerAddress;
 109     ES.reportError(Sym.takeError());
 110     return ErrorHandlerAddress;
 111   }
 112 }
 113
 114 Expected<std::unique_ptr<JITCompileCallbackManager>>
 115 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
 116                                   JITTargetAddress ErrorHandlerAddress) {
 117   switch (T.getArch()) {
 118   default:
 119     return make_error<StringError>(
 120         std::string("No callback manager available for ") + T.str(),
 121         inconvertibleErrorCode());
 122   case Triple::aarch64: {
 123     typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
 124     return CCMgrT::Create(ES, ErrorHandlerAddress);
 125     }
 126
 127     case Triple::x86: {
 128       typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
 129       return CCMgrT::Create(ES, ErrorHandlerAddress);
 130     }
 131
 132     case Triple::mips: {
 133       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
 134       return CCMgrT::Create(ES, ErrorHandlerAddress);
 135     }
 136     case Triple::mipsel: {
 137       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
 138       return CCMgrT::Create(ES, ErrorHandlerAddress);
 139     }
 140
 141     case Triple::mips64:
 142     case Triple::mips64el: {
 143       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
 144       return CCMgrT::Create(ES, ErrorHandlerAddress);
 145     }
 146
 147     case Triple::x86_64: {
 148       if ( T.getOS() == Triple::OSType::Win32 ) {
 149         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
 150         return CCMgrT::Create(ES, ErrorHandlerAddress);
 151       } else {
 152         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
 153         return CCMgrT::Create(ES, ErrorHandlerAddress);
 154       }
 155     }
 156
 157   }
 158 }
 159
 160 std::function<std::unique_ptr<IndirectStubsManager>()>
 161 createLocalIndirectStubsManagerBuilder(const Triple &T) {
 162   switch (T.getArch()) {
 163     default:
 164       return [](){
 165         return llvm::make_unique<
 166                        orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
 167       };
 168
 169     case Triple::aarch64:
 170       return [](){
 171         return llvm::make_unique<
 172                        orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
 173       };
 174
 175     case Triple::x86:
 176       return [](){
 177         return llvm::make_unique<
 178                        orc::LocalIndirectStubsManager<orc::OrcI386>>();
 179       };
 180
 181     case Triple::mips:
 182       return [](){
 183           return llvm::make_unique<
 184                       orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
 185       };
 186
 187     case Triple::mipsel:
 188       return [](){
 189           return llvm::make_unique<
 190                       orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
 191       };
 192
 193     case Triple::mips64:
 194     case Triple::mips64el:
 195       return [](){
 196           return llvm::make_unique<
 197                       orc::LocalIndirectStubsManager<orc::OrcMips64>>();
 198       };
 199
 200     case Triple::x86_64:
 201       if (T.getOS() == Triple::OSType::Win32) {
 202         return [](){
 203           return llvm::make_unique<
 204                      orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
 205         };
 206       } else {
 207         return [](){
 208           return llvm::make_unique<
 209                      orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
 210         };
 211       }
 212
 213   }
 214 }
 215
 216 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
 217   Constant *AddrIntVal =
 218     ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
 219   Constant *AddrPtrVal =
 220     ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
 221                           PointerType::get(&FT, 0));
 222   return AddrPtrVal;
 223 }
 224
 225 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
 226                                   const Twine &Name, Constant *Initializer) {
 227   auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
 228                                Initializer, Name, nullptr,
 229                                GlobalValue::NotThreadLocal, 0, true);
 230   IP->setVisibility(GlobalValue::HiddenVisibility);
 231   return IP;
 232 }
 233
 234 void makeStub(Function &F, Value &ImplPointer) {
 235   assert(F.isDeclaration() && "Can't turn a definition into a stub.");
 236   assert(F.getParent() && "Function isn't in a module.");
 237   Module &M = *F.getParent();
 238   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
 239   IRBuilder<> Builder(EntryBlock);
 240   LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
 241   std::vector<Value*> CallArgs;
 242   for (auto &A : F.args())
 243     CallArgs.push_back(&A);
 244   CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
 245   Call->setTailCall();
 246   Call->setAttributes(F.getAttributes());
 247   if (F.getReturnType()->isVoidTy())
 248     Builder.CreateRetVoid();
 249   else
 250     Builder.CreateRet(Call);
 251 }
 252
 253 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
 254   std::vector<GlobalValue *> PromotedGlobals;
 255
 256   for (auto &GV : M.global_values()) {
 257     bool Promoted = true;
 258
 259     // Rename if necessary.
 260     if (!GV.hasName())
 261       GV.setName("__orc_anon." + Twine(NextId++));
 262     else if (GV.getName().startswith("\01L"))
 263       GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
 264     else if (GV.hasLocalLinkage())
 265       GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
 266     else
 267       Promoted = false;
 268
 269     if (GV.hasLocalLinkage()) {
 270       GV.setLinkage(GlobalValue::ExternalLinkage);
 271       GV.setVisibility(GlobalValue::HiddenVisibility);
 272       Promoted = true;
 273     }
 274     GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
 275
 276     if (Promoted)
 277       PromotedGlobals.push_back(&GV);
 278   }
 279
 280   return PromotedGlobals;
 281 }
 282
 283 Function* cloneFunctionDecl(Module &Dst, const Function &F,
 284                             ValueToValueMapTy *VMap) {
 285   Function *NewF =
 286     Function::Create(cast<FunctionType>(F.getValueType()),
 287                      F.getLinkage(), F.getName(), &Dst);
 288   NewF->copyAttributesFrom(&F);
 289
 290   if (VMap) {
 291     (*VMap)[&F] = NewF;
 292     auto NewArgI = NewF->arg_begin();
 293     for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
 294          ++ArgI, ++NewArgI)
 295       (*VMap)[&*ArgI] = &*NewArgI;
 296   }
 297
 298   return NewF;
 299 }
 300
 301 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
 302                       ValueMaterializer *Materializer,
 303                       Function *NewF) {
 304   assert(!OrigF.isDeclaration() && "Nothing to move");
 305   if (!NewF)
 306     NewF = cast<Function>(VMap[&OrigF]);
 307   else
 308     assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
 309   assert(NewF && "Function mapping missing from VMap.");
 310   assert(NewF->getParent() != OrigF.getParent() &&
 311          "moveFunctionBody should only be used to move bodies between "
 312          "modules.");
 313
 314   SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
 315   CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns,
 316                     "", nullptr, nullptr, Materializer);
 317   OrigF.deleteBody();
 318 }
 319
 320 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
 321                                         ValueToValueMapTy *VMap) {
 322   GlobalVariable *NewGV = new GlobalVariable(
 323       Dst, GV.getValueType(), GV.isConstant(),
 324       GV.getLinkage(), nullptr, GV.getName(), nullptr,
 325       GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
 326   NewGV->copyAttributesFrom(&GV);
 327   if (VMap)
 328     (*VMap)[&GV] = NewGV;
 329   return NewGV;
 330 }
 331
 332 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
 333                                    ValueToValueMapTy &VMap,
 334                                    ValueMaterializer *Materializer,
 335                                    GlobalVariable *NewGV) {
 336   assert(OrigGV.hasInitializer() && "Nothing to move");
 337   if (!NewGV)
 338     NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
 339   else
 340     assert(VMap[&OrigGV] == NewGV &&
 341            "Incorrect global variable mapping in VMap.");
 342   assert(NewGV->getParent() != OrigGV.getParent() &&
 343          "moveGlobalVariableInitializer should only be used to move "
 344          "initializers between modules");
 345
 346   NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
 347                                  nullptr, Materializer));
 348 }
 349
 350 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
 351                                   ValueToValueMapTy &VMap) {
 352   assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
 353   auto *NewA = GlobalAlias::create(OrigA.getValueType(),
 354                                    OrigA.getType()->getPointerAddressSpace(),
 355                                    OrigA.getLinkage(), OrigA.getName(), &Dst);
 356   NewA->copyAttributesFrom(&OrigA);
 357   VMap[&OrigA] = NewA;
 358   return NewA;
 359 }
 360
 361 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
 362                               ValueToValueMapTy &VMap) {
 363   auto *MFs = Src.getModuleFlagsMetadata();
 364   if (!MFs)
 365     return;
 366   for (auto *MF : MFs->operands())
 367     Dst.addModuleFlag(MapMetadata(MF, VMap));
 368 }
 369
 370 } // End namespace orc.
 371 } // End namespace llvm.