contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h

   1 //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
  10 #define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
  11
  12 #include "llvm-c/OrcBindings.h"
  13 #include "llvm-c/TargetMachine.h"
  14 #include "llvm/ADT/STLExtras.h"
  15 #include "llvm/ADT/StringRef.h"
  16 #include "llvm/ExecutionEngine/JITSymbol.h"
  17 #include "llvm/ExecutionEngine/JITEventListener.h"
  18 #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
  19 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
  20 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
  21 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
  22 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
  23 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  24 #include "llvm/ExecutionEngine/RuntimeDyld.h"
  25 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
  26 #include "llvm/IR/DataLayout.h"
  27 #include "llvm/IR/Mangler.h"
  28 #include "llvm/IR/Module.h"
  29 #include "llvm/Support/CBindingWrapping.h"
  30 #include "llvm/Support/Error.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include "llvm/Target/TargetMachine.h"
  33 #include <algorithm>
  34 #include <cstdint>
  35 #include <functional>
  36 #include <map>
  37 #include <memory>
  38 #include <set>
  39 #include <string>
  40 #include <vector>
  41
  42 namespace llvm {
  43
  44 class OrcCBindingsStack;
  45
  46 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
  47 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
  48
  49 namespace detail {
  50
  51 // FIXME: Kill this off once the Layer concept becomes an interface.
  52 class GenericLayer {
  53 public:
  54   virtual ~GenericLayer() = default;
  55
  56   virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
  57                                  bool ExportedSymbolsOnly) = 0;
  58   virtual Error removeModule(orc::VModuleKey K) = 0;
  59   };
  60
  61   template <typename LayerT> class GenericLayerImpl : public GenericLayer {
  62   public:
  63     GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
  64
  65     JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
  66                            bool ExportedSymbolsOnly) override {
  67       return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
  68     }
  69
  70     Error removeModule(orc::VModuleKey K) override {
  71       return Layer.removeModule(K);
  72     }
  73
  74   private:
  75     LayerT &Layer;
  76   };
  77
  78   template <>
  79   class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
  80   private:
  81     using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
  82   public:
  83     GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
  84
  85     JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
  86                            bool ExportedSymbolsOnly) override {
  87       return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
  88     }
  89
  90     Error removeModule(orc::VModuleKey K) override {
  91       return Layer.removeObject(K);
  92     }
  93
  94   private:
  95     LayerT &Layer;
  96   };
  97
  98   template <typename LayerT>
  99   std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
 100     return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer);
 101   }
 102
 103 } // end namespace detail
 104
 105 class OrcCBindingsStack {
 106 public:
 107
 108   using CompileCallbackMgr = orc::JITCompileCallbackManager;
 109   using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
 110   using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
 111   using CODLayerT =
 112         orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
 113
 114   using CallbackManagerBuilder =
 115       std::function<std::unique_ptr<CompileCallbackMgr>()>;
 116
 117   using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
 118
 119 private:
 120
 121   using OwningObject = object::OwningBinary<object::ObjectFile>;
 122
 123   class CBindingsResolver : public orc::SymbolResolver {
 124   public:
 125     CBindingsResolver(OrcCBindingsStack &Stack,
 126                       LLVMOrcSymbolResolverFn ExternalResolver,
 127                       void *ExternalResolverCtx)
 128         : Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
 129           ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
 130
 131     orc::SymbolNameSet
 132     getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
 133       orc::SymbolNameSet Result;
 134
 135       for (auto &S : Symbols) {
 136         if (auto Sym = findSymbol(*S)) {
 137           if (!Sym.getFlags().isStrong())
 138             Result.insert(S);
 139         } else if (auto Err = Sym.takeError()) {
 140           Stack.reportError(std::move(Err));
 141           return orc::SymbolNameSet();
 142         }
 143       }
 144
 145       return Result;
 146     }
 147
 148     orc::SymbolNameSet
 149     lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
 150            orc::SymbolNameSet Symbols) override {
 151       orc::SymbolNameSet UnresolvedSymbols;
 152
 153       for (auto &S : Symbols) {
 154         if (auto Sym = findSymbol(*S)) {
 155           if (auto Addr = Sym.getAddress()) {
 156             Query->notifySymbolMetRequiredState(
 157                 S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
 158           } else {
 159             Stack.ES.legacyFailQuery(*Query, Addr.takeError());
 160             return orc::SymbolNameSet();
 161           }
 162         } else if (auto Err = Sym.takeError()) {
 163           Stack.ES.legacyFailQuery(*Query, std::move(Err));
 164           return orc::SymbolNameSet();
 165         } else
 166           UnresolvedSymbols.insert(S);
 167       }
 168
 169       if (Query->isComplete())
 170         Query->handleComplete();
 171
 172       return UnresolvedSymbols;
 173     }
 174
 175   private:
 176     JITSymbol findSymbol(const std::string &Name) {
 177       // Search order:
 178       // 1. JIT'd symbols.
 179       // 2. Runtime overrides.
 180       // 3. External resolver (if present).
 181
 182       if (Stack.CODLayer) {
 183         if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
 184           return Sym;
 185         else if (auto Err = Sym.takeError())
 186           return Sym.takeError();
 187       } else {
 188         if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
 189           return Sym;
 190         else if (auto Err = Sym.takeError())
 191           return Sym.takeError();
 192       }
 193
 194       if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
 195         return Sym;
 196
 197       if (ExternalResolver)
 198         return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
 199                          JITSymbolFlags::Exported);
 200
 201       return JITSymbol(nullptr);
 202     }
 203
 204     OrcCBindingsStack &Stack;
 205     LLVMOrcSymbolResolverFn ExternalResolver;
 206     void *ExternalResolverCtx = nullptr;
 207   };
 208
 209 public:
 210   OrcCBindingsStack(TargetMachine &TM,
 211                     IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
 212       : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
 213         IndirectStubsMgr(IndirectStubsMgrBuilder()),
 214         ObjectLayer(
 215             AcknowledgeORCv1Deprecation, ES,
 216             [this](orc::VModuleKey K) {
 217               auto ResolverI = Resolvers.find(K);
 218               assert(ResolverI != Resolvers.end() &&
 219                      "No resolver for module K");
 220               auto Resolver = std::move(ResolverI->second);
 221               Resolvers.erase(ResolverI);
 222               return ObjLayerT::Resources{
 223                   std::make_shared<SectionMemoryManager>(), Resolver};
 224             },
 225             nullptr,
 226             [this](orc::VModuleKey K, const object::ObjectFile &Obj,
 227                    const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
 228               this->notifyFinalized(K, Obj, LoadedObjInfo);
 229             },
 230             [this](orc::VModuleKey K, const object::ObjectFile &Obj) {
 231               this->notifyFreed(K, Obj);
 232             }),
 233         CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
 234                      orc::SimpleCompiler(TM)),
 235         CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
 236                                 std::move(IndirectStubsMgrBuilder), Resolvers)),
 237         CXXRuntimeOverrides(
 238             AcknowledgeORCv1Deprecation,
 239             [this](const std::string &S) { return mangle(S); }) {}
 240
 241   Error shutdown() {
 242     // Run any destructors registered with __cxa_atexit.
 243     CXXRuntimeOverrides.runDestructors();
 244     // Run any IR destructors.
 245     for (auto &DtorRunner : IRStaticDestructorRunners)
 246       if (auto Err = DtorRunner.runViaLayer(*this))
 247         return Err;
 248     return Error::success();
 249   }
 250
 251   std::string mangle(StringRef Name) {
 252     std::string MangledName;
 253     {
 254       raw_string_ostream MangledNameStream(MangledName);
 255       Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
 256     }
 257     return MangledName;
 258   }
 259
 260   template <typename PtrTy>
 261   static PtrTy fromTargetAddress(JITTargetAddress Addr) {
 262     return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
 263   }
 264
 265   Expected<JITTargetAddress>
 266   createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
 267                             void *CallbackCtx) {
 268     auto WrappedCallback = [=]() -> JITTargetAddress {
 269       return Callback(wrap(this), CallbackCtx);
 270     };
 271
 272     return CCMgr->getCompileCallback(std::move(WrappedCallback));
 273   }
 274
 275   Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
 276     return IndirectStubsMgr->createStub(StubName, Addr,
 277                                         JITSymbolFlags::Exported);
 278   }
 279
 280   Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
 281     return IndirectStubsMgr->updatePointer(Name, Addr);
 282   }
 283
 284   template <typename LayerT>
 285   Expected<orc::VModuleKey>
 286   addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
 287               std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
 288               LLVMOrcSymbolResolverFn ExternalResolver,
 289               void *ExternalResolverCtx) {
 290
 291     // Attach a data-layout if one isn't already present.
 292     if (M->getDataLayout().isDefault())
 293       M->setDataLayout(DL);
 294
 295     // Record the static constructors and destructors. We have to do this before
 296     // we hand over ownership of the module to the JIT.
 297     std::vector<std::string> CtorNames, DtorNames;
 298     for (auto Ctor : orc::getConstructors(*M))
 299       CtorNames.push_back(mangle(Ctor.Func->getName()));
 300     for (auto Dtor : orc::getDestructors(*M))
 301       DtorNames.push_back(mangle(Dtor.Func->getName()));
 302
 303     // Add the module to the JIT.
 304     auto K = ES.allocateVModule();
 305     Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
 306                                                        ExternalResolverCtx);
 307     if (auto Err = Layer.addModule(K, std::move(M)))
 308       return std::move(Err);
 309
 310     KeyLayers[K] = detail::createGenericLayer(Layer);
 311
 312     // Run the static constructors, and save the static destructor runner for
 313     // execution when the JIT is torn down.
 314     orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(
 315         AcknowledgeORCv1Deprecation, std::move(CtorNames), K);
 316     if (auto Err = CtorRunner.runViaLayer(*this))
 317       return std::move(Err);
 318
 319     IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K);
 320
 321     return K;
 322   }
 323
 324   Expected<orc::VModuleKey>
 325   addIRModuleEager(std::unique_ptr<Module> M,
 326                    LLVMOrcSymbolResolverFn ExternalResolver,
 327                    void *ExternalResolverCtx) {
 328     return addIRModule(CompileLayer, std::move(M),
 329                        llvm::make_unique<SectionMemoryManager>(),
 330                        std::move(ExternalResolver), ExternalResolverCtx);
 331   }
 332
 333   Expected<orc::VModuleKey>
 334   addIRModuleLazy(std::unique_ptr<Module> M,
 335                   LLVMOrcSymbolResolverFn ExternalResolver,
 336                   void *ExternalResolverCtx) {
 337     if (!CODLayer)
 338       return make_error<StringError>("Can not add lazy module: No compile "
 339                                      "callback manager available",
 340                                      inconvertibleErrorCode());
 341
 342     return addIRModule(*CODLayer, std::move(M),
 343                        llvm::make_unique<SectionMemoryManager>(),
 344                        std::move(ExternalResolver), ExternalResolverCtx);
 345   }
 346
 347   Error removeModule(orc::VModuleKey K) {
 348     // FIXME: Should error release the module key?
 349     if (auto Err = KeyLayers[K]->removeModule(K))
 350       return Err;
 351     ES.releaseVModule(K);
 352     KeyLayers.erase(K);
 353     return Error::success();
 354   }
 355
 356   Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
 357                                       LLVMOrcSymbolResolverFn ExternalResolver,
 358                                       void *ExternalResolverCtx) {
 359     if (auto Obj = object::ObjectFile::createObjectFile(
 360             ObjBuffer->getMemBufferRef())) {
 361
 362       auto K = ES.allocateVModule();
 363       Resolvers[K] = std::make_shared<CBindingsResolver>(
 364           *this, ExternalResolver, ExternalResolverCtx);
 365
 366       if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
 367         return std::move(Err);
 368
 369       KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
 370
 371       return K;
 372     } else
 373       return Obj.takeError();
 374   }
 375
 376   JITSymbol findSymbol(const std::string &Name,
 377                                  bool ExportedSymbolsOnly) {
 378     if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
 379       return Sym;
 380     if (CODLayer)
 381       return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
 382     return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
 383   }
 384
 385   JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
 386                          bool ExportedSymbolsOnly) {
 387     assert(KeyLayers.count(K) && "looking up symbol in unknown module");
 388     return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
 389   }
 390
 391   Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
 392                                                bool ExportedSymbolsOnly) {
 393     if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
 394       // Successful lookup, non-null symbol:
 395       if (auto AddrOrErr = Sym.getAddress())
 396         return *AddrOrErr;
 397       else
 398         return AddrOrErr.takeError();
 399     } else if (auto Err = Sym.takeError()) {
 400       // Lookup failure - report error.
 401       return std::move(Err);
 402     }
 403
 404     // No symbol not found. Return 0.
 405     return 0;
 406   }
 407
 408   Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
 409                                                  const std::string &Name,
 410                                                  bool ExportedSymbolsOnly) {
 411     if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
 412       // Successful lookup, non-null symbol:
 413       if (auto AddrOrErr = Sym.getAddress())
 414         return *AddrOrErr;
 415       else
 416         return AddrOrErr.takeError();
 417     } else if (auto Err = Sym.takeError()) {
 418       // Lookup failure - report error.
 419       return std::move(Err);
 420     }
 421
 422     // Symbol not found. Return 0.
 423     return 0;
 424   }
 425
 426   const std::string &getErrorMessage() const { return ErrMsg; }
 427
 428   void RegisterJITEventListener(JITEventListener *L) {
 429     if (!L)
 430       return;
 431     EventListeners.push_back(L);
 432   }
 433
 434   void UnregisterJITEventListener(JITEventListener *L) {
 435     if (!L)
 436       return;
 437
 438     auto I = find(reverse(EventListeners), L);
 439     if (I != EventListeners.rend()) {
 440       std::swap(*I, EventListeners.back());
 441       EventListeners.pop_back();
 442     }
 443   }
 444
 445 private:
 446   using ResolverMap =
 447       std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
 448
 449   static std::unique_ptr<CompileCallbackMgr>
 450   createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
 451     auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
 452     if (!CCMgr) {
 453       // FIXME: It would be good if we could report this somewhere, but we do
 454       //        have an instance yet.
 455       logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
 456       return nullptr;
 457     }
 458     return std::move(*CCMgr);
 459   }
 460
 461   static std::unique_ptr<CODLayerT>
 462   createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
 463                  CompileCallbackMgr *CCMgr,
 464                  IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
 465                  ResolverMap &Resolvers) {
 466     // If there is no compile callback manager available we can not create a
 467     // compile on demand layer.
 468     if (!CCMgr)
 469       return nullptr;
 470
 471     return llvm::make_unique<CODLayerT>(
 472         AcknowledgeORCv1Deprecation, ES, CompileLayer,
 473         [&Resolvers](orc::VModuleKey K) {
 474           auto ResolverI = Resolvers.find(K);
 475           assert(ResolverI != Resolvers.end() && "No resolver for module K");
 476           return ResolverI->second;
 477         },
 478         [&Resolvers](orc::VModuleKey K,
 479                      std::shared_ptr<orc::SymbolResolver> Resolver) {
 480           assert(!Resolvers.count(K) && "Resolver already present");
 481           Resolvers[K] = std::move(Resolver);
 482         },
 483         [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
 484         std::move(IndirectStubsMgrBuilder), false);
 485   }
 486
 487   void reportError(Error Err) {
 488     // FIXME: Report errors on the execution session.
 489     logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
 490   };
 491
 492   void notifyFinalized(orc::VModuleKey K,
 493                        const object::ObjectFile &Obj,
 494                        const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
 495     uint64_t Key = static_cast<uint64_t>(
 496         reinterpret_cast<uintptr_t>(Obj.getData().data()));
 497     for (auto &Listener : EventListeners)
 498       Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
 499   }
 500
 501   void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
 502     uint64_t Key = static_cast<uint64_t>(
 503         reinterpret_cast<uintptr_t>(Obj.getData().data()));
 504     for (auto &Listener : EventListeners)
 505       Listener->notifyFreeingObject(Key);
 506   }
 507
 508   orc::ExecutionSession ES;
 509   std::unique_ptr<CompileCallbackMgr> CCMgr;
 510
 511   std::vector<JITEventListener *> EventListeners;
 512
 513   DataLayout DL;
 514   SectionMemoryManager CCMgrMemMgr;
 515
 516   std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
 517
 518   ObjLayerT ObjectLayer;
 519   CompileLayerT CompileLayer;
 520   std::unique_ptr<CODLayerT> CODLayer;
 521
 522   std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
 523
 524   orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
 525   std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
 526   std::string ErrMsg;
 527
 528   ResolverMap Resolvers;
 529 };
 530
 531 } // end namespace llvm
 532
 533 #endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H