1 //===- elfnix_platform.cpp ------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
11 //===----------------------------------------------------------------------===//
13 #include "elfnix_platform.h"
16 #include "wrapper_function_utils.h"
21 #include <unordered_map>
24 using namespace __orc_rt;
25 using namespace __orc_rt::elfnix;
27 // Declare function tags for functions in the JIT process.
28 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
29 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
30 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
32 // eh-frame registration functions.
33 // We expect these to be available for all processes.
34 extern "C" void __register_frame(const void *);
35 extern "C" void __deregister_frame(const void *);
39 Error validatePointerSectionExtent(const char *SectionName,
40 const ExecutorAddrRange &SE) {
41 if (SE.size().getValue() % sizeof(uintptr_t)) {
42 std::ostringstream ErrMsg;
43 ErrMsg << std::hex << "Size of " << SectionName << " 0x"
44 << SE.Start.getValue() << " -- 0x" << SE.End.getValue()
45 << " is not a pointer multiple";
46 return make_error<StringError>(ErrMsg.str());
48 return Error::success();
51 Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
52 const ELFNixJITDylibInitializers &MOJDIs) {
54 for (const auto &ModInits : InitArraySections) {
55 if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
58 using InitFunc = void (*)();
59 for (auto *Init : ModInits.toSpan<InitFunc>())
63 return Error::success();
66 unsigned long Key = 0;
67 unsigned long DataAddress = 0;
70 class ELFNixPlatformRuntimeState {
77 using AtExitsVector = std::vector<AtExitEntry>;
79 struct PerJITDylibState {
80 void *Header = nullptr;
82 bool AllowReinitialization = false;
83 AtExitsVector AtExits;
87 static void initialize(void *DSOHandle);
88 static ELFNixPlatformRuntimeState &get();
89 static void destroy();
91 ELFNixPlatformRuntimeState(void *DSOHandle)
92 : PlatformJDDSOHandle(DSOHandle) {}
94 // Delete copy and move constructors.
95 ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
96 ELFNixPlatformRuntimeState &
97 operator=(const ELFNixPlatformRuntimeState &) = delete;
98 ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
99 ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
101 Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
102 Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
104 const char *dlerror();
105 void *dlopen(string_view Name, int Mode);
106 int dlclose(void *DSOHandle);
107 void *dlsym(void *DSOHandle, string_view Symbol);
109 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
110 void runAtExits(void *DSOHandle);
112 /// Returns the base address of the section containing ThreadData.
113 Expected<std::pair<const char *, size_t>>
114 getThreadDataSectionFor(const char *ThreadData);
116 void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
119 PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
120 PerJITDylibState *getJITDylibStateByName(string_view Path);
122 getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
124 Error registerThreadDataSection(span<const char> ThreadDataSection);
126 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
129 Expected<ELFNixJITDylibInitializerSequence>
130 getJITDylibInitializersByName(string_view Path);
131 Expected<void *> dlopenInitialize(string_view Path, int Mode);
132 Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
134 static ELFNixPlatformRuntimeState *MOPS;
136 using InitSectionHandler =
137 Error (*)(const std::vector<ExecutorAddrRange> &Sections,
138 const ELFNixJITDylibInitializers &MOJDIs);
139 const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
140 {{".init_array", runInitArray}};
142 void *PlatformJDDSOHandle;
144 // FIXME: Move to thread-state.
145 std::string DLFcnError;
147 std::recursive_mutex JDStatesMutex;
148 std::unordered_map<void *, PerJITDylibState> JDStates;
149 std::unordered_map<std::string, void *> JDNameToHeader;
151 std::mutex ThreadDataSectionsMutex;
152 std::map<const char *, size_t> ThreadDataSections;
155 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
157 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
158 assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
159 MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
162 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
163 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
167 void ELFNixPlatformRuntimeState::destroy() {
168 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
172 Error ELFNixPlatformRuntimeState::registerObjectSections(
173 ELFNixPerObjectSectionsToRegister POSR) {
174 if (POSR.EHFrameSection.Start)
175 __register_frame(POSR.EHFrameSection.Start.toPtr<const char *>());
177 if (POSR.ThreadDataSection.Start) {
178 if (auto Err = registerThreadDataSection(
179 POSR.ThreadDataSection.toSpan<const char>()))
183 return Error::success();
186 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
187 ELFNixPerObjectSectionsToRegister POSR) {
188 if (POSR.EHFrameSection.Start)
189 __deregister_frame(POSR.EHFrameSection.Start.toPtr<const char *>());
191 return Error::success();
194 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
196 void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
197 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
199 // Use fast path if all JITDylibs are already loaded and don't require
200 // re-running initializers.
201 if (auto *JDS = getJITDylibStateByName(Path)) {
202 if (!JDS->AllowReinitialization) {
208 auto H = dlopenInitialize(Path, Mode);
210 DLFcnError = toString(H.takeError());
217 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
218 runAtExits(DSOHandle);
222 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
223 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
225 DLFcnError = toString(Addr.takeError());
229 return Addr->toPtr<void *>();
232 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
234 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
235 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
236 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
237 assert(JDS && "JITDylib state not initialized");
238 JDS->AtExits.push_back({F, Arg});
242 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
243 // FIXME: Should atexits be allowed to run concurrently with access to
247 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
248 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
249 assert(JDS && "JITDlybi state not initialized");
250 std::swap(V, JDS->AtExits);
260 Expected<std::pair<const char *, size_t>>
261 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
262 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
263 auto I = ThreadDataSections.upper_bound(ThreadData);
264 // Check that we have a valid entry conovering this address.
265 if (I == ThreadDataSections.begin())
266 return make_error<StringError>("No thread local data section for key");
268 if (ThreadData >= I->first + I->second)
269 return make_error<StringError>("No thread local data section for key");
273 ELFNixPlatformRuntimeState::PerJITDylibState *
274 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
275 auto I = JDStates.find(DSOHandle);
276 if (I == JDStates.end())
281 ELFNixPlatformRuntimeState::PerJITDylibState *
282 ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
283 // FIXME: Avoid creating string copy here.
284 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
285 if (I == JDNameToHeader.end())
288 auto J = JDStates.find(H);
289 assert(J != JDStates.end() &&
290 "JITDylib has name map entry but no header map entry");
294 ELFNixPlatformRuntimeState::PerJITDylibState &
295 ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
296 ELFNixJITDylibInitializers &MOJDIs) {
297 void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
299 auto &JDS = JDStates[Header];
301 // If this entry hasn't been created yet.
303 assert(!JDNameToHeader.count(MOJDIs.Name) &&
304 "JITDylib has header map entry but no name map entry");
305 JDNameToHeader[MOJDIs.Name] = Header;
312 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
313 span<const char> ThreadDataSection) {
314 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
315 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
316 if (I != ThreadDataSections.begin()) {
317 auto J = std::prev(I);
318 if (J->first + J->second > ThreadDataSection.data())
319 return make_error<StringError>("Overlapping .tdata sections");
321 ThreadDataSections.insert(
322 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
323 return Error::success();
326 Expected<ExecutorAddr>
327 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
329 Expected<ExecutorAddr> Result((ExecutorAddr()));
330 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
331 SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
333 ExecutorAddr::fromPtr(DSOHandle),
335 return std::move(Err);
339 Expected<ELFNixJITDylibInitializerSequence>
340 ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
341 Expected<ELFNixJITDylibInitializerSequence> Result(
342 (ELFNixJITDylibInitializerSequence()));
343 std::string PathStr(Path.data(), Path.size());
345 WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
346 SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
348 return std::move(Err);
352 Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
354 // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
355 // reinitialization. We need to call in to the JIT to see if there's any new
357 auto InitSeq = getJITDylibInitializersByName(Path);
359 return InitSeq.takeError();
361 // Init sequences should be non-empty.
362 if (InitSeq->empty())
363 return make_error<StringError>(
364 "__orc_rt_elfnix_get_initializers returned an "
365 "empty init sequence");
367 // Otherwise register and run initializers for each JITDylib.
368 for (auto &MOJDIs : *InitSeq)
369 if (auto Err = initializeJITDylib(MOJDIs))
370 return std::move(Err);
372 // Return the header for the last item in the list.
373 auto *JDS = getJITDylibStateByHeaderAddr(
374 InitSeq->back().DSOHandleAddress.toPtr<void *>());
375 assert(JDS && "Missing state entry for JD");
379 Error ELFNixPlatformRuntimeState::initializeJITDylib(
380 ELFNixJITDylibInitializers &MOJDIs) {
382 auto &JDS = getOrCreateJITDylibState(MOJDIs);
385 for (auto &KV : InitSections) {
386 const auto &Name = KV.first;
387 const auto &Handler = KV.second;
388 auto I = MOJDIs.InitSections.find(Name);
389 if (I != MOJDIs.InitSections.end()) {
390 if (auto Err = Handler(I->second, MOJDIs))
395 return Error::success();
397 class ELFNixPlatformRuntimeTLVManager {
399 void *getInstance(const char *ThreadData);
402 std::unordered_map<const char *, char *> Instances;
403 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
406 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
407 auto I = Instances.find(ThreadData);
408 if (I != Instances.end())
411 ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
413 __orc_rt_log_error(toString(TDS.takeError()).c_str());
417 auto &Allocated = AllocatedSections[TDS->first];
419 Allocated = std::make_unique<char[]>(TDS->second);
420 memcpy(Allocated.get(), TDS->first, TDS->second);
422 size_t ThreadDataDelta = ThreadData - TDS->first;
423 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
425 char *Instance = Allocated.get() + ThreadDataDelta;
426 Instances[ThreadData] = Instance;
430 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
431 delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
434 } // end anonymous namespace
436 //------------------------------------------------------------------------------
438 //------------------------------------------------------------------------------
440 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
441 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
442 return WrapperFunction<void(uint64_t)>::handle(
444 [](uint64_t &DSOHandle) {
445 ELFNixPlatformRuntimeState::initialize(
446 reinterpret_cast<void *>(DSOHandle));
451 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
452 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
453 ELFNixPlatformRuntimeState::destroy();
454 return WrapperFunctionResult().release();
457 /// Wrapper function for registering metadata on a per-object basis.
458 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
459 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
460 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
461 handle(ArgData, ArgSize,
462 [](ELFNixPerObjectSectionsToRegister &POSR) {
463 return ELFNixPlatformRuntimeState::get().registerObjectSections(
469 /// Wrapper for releasing per-object metadat.
470 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
471 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
472 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
473 handle(ArgData, ArgSize,
474 [](ELFNixPerObjectSectionsToRegister &POSR) {
475 return ELFNixPlatformRuntimeState::get()
476 .deregisterObjectSections(std::move(POSR));
481 //------------------------------------------------------------------------------
483 //------------------------------------------------------------------------------
485 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
486 auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
487 pthread_getspecific(D->Key));
489 TLVMgr = new ELFNixPlatformRuntimeTLVManager();
490 if (pthread_setspecific(D->Key, TLVMgr)) {
491 __orc_rt_log_error("Call to pthread_setspecific failed");
495 return TLVMgr->getInstance(
496 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
499 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
500 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
501 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
503 []() -> Expected<uint64_t> {
505 if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
506 __orc_rt_log_error("Call to pthread_key_create failed");
507 return make_error<StringError>(strerror(Err));
509 return static_cast<uint64_t>(Key);
514 //------------------------------------------------------------------------------
515 // cxa_atexit support
516 //------------------------------------------------------------------------------
518 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
520 return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
524 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
525 auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
526 return ELFNixPlatformRuntimeState::get().registerAtExit(
527 func, NULL, PlatformRTState.getPlatformJDDSOHandle());
530 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
531 ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
534 //------------------------------------------------------------------------------
535 // JIT'd dlfcn alternatives.
536 //------------------------------------------------------------------------------
538 const char *__orc_rt_elfnix_jit_dlerror() {
539 return ELFNixPlatformRuntimeState::get().dlerror();
542 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
543 return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
546 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
547 return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
550 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
551 return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
554 //------------------------------------------------------------------------------
555 // ELFNix Run Program
556 //------------------------------------------------------------------------------
558 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
559 const char *JITDylibName, const char *EntrySymbolName, int argc,
561 using MainTy = int (*)(int, char *[]);
563 void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
564 __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
566 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
571 reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
574 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
578 int Result = Main(argc, argv);
580 if (__orc_rt_elfnix_jit_dlclose(H) == -1)
581 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());