]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - contrib/llvm-project/compiler-rt/lib/orc/elfnix_platform.cpp
zfs: merge openzfs/zfs@92e0d9d18 (zfs-2.1-release) into stable/13
[FreeBSD/FreeBSD.git] / contrib / llvm-project / compiler-rt / lib / orc / elfnix_platform.cpp
1 //===- elfnix_platform.cpp ------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "elfnix_platform.h"
14 #include "common.h"
15 #include "error.h"
16 #include "wrapper_function_utils.h"
17
18 #include <map>
19 #include <mutex>
20 #include <sstream>
21 #include <unordered_map>
22 #include <vector>
23
24 using namespace __orc_rt;
25 using namespace __orc_rt::elfnix;
26
27 // Declare function tags for functions in the JIT process.
28 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
29 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
30 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
31
32 // eh-frame registration functions.
33 // We expect these to be available for all processes.
34 extern "C" void __register_frame(const void *);
35 extern "C" void __deregister_frame(const void *);
36
37 namespace {
38
39 Error validatePointerSectionExtent(const char *SectionName,
40                                    const ExecutorAddrRange &SE) {
41   if (SE.size().getValue() % sizeof(uintptr_t)) {
42     std::ostringstream ErrMsg;
43     ErrMsg << std::hex << "Size of " << SectionName << " 0x"
44            << SE.Start.getValue() << " -- 0x" << SE.End.getValue()
45            << " is not a pointer multiple";
46     return make_error<StringError>(ErrMsg.str());
47   }
48   return Error::success();
49 }
50
51 Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
52                    const ELFNixJITDylibInitializers &MOJDIs) {
53
54   for (const auto &ModInits : InitArraySections) {
55     if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
56       return Err;
57
58     using InitFunc = void (*)();
59     for (auto *Init : ModInits.toSpan<InitFunc>())
60       (*Init)();
61   }
62
63   return Error::success();
64 }
65 struct TLSInfoEntry {
66   unsigned long Key = 0;
67   unsigned long DataAddress = 0;
68 };
69
70 class ELFNixPlatformRuntimeState {
71 private:
72   struct AtExitEntry {
73     void (*Func)(void *);
74     void *Arg;
75   };
76
77   using AtExitsVector = std::vector<AtExitEntry>;
78
79   struct PerJITDylibState {
80     void *Header = nullptr;
81     size_t RefCount = 0;
82     bool AllowReinitialization = false;
83     AtExitsVector AtExits;
84   };
85
86 public:
87   static void initialize(void *DSOHandle);
88   static ELFNixPlatformRuntimeState &get();
89   static void destroy();
90
91   ELFNixPlatformRuntimeState(void *DSOHandle)
92       : PlatformJDDSOHandle(DSOHandle) {}
93
94   // Delete copy and move constructors.
95   ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
96   ELFNixPlatformRuntimeState &
97   operator=(const ELFNixPlatformRuntimeState &) = delete;
98   ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
99   ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
100
101   Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
102   Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
103
104   const char *dlerror();
105   void *dlopen(string_view Name, int Mode);
106   int dlclose(void *DSOHandle);
107   void *dlsym(void *DSOHandle, string_view Symbol);
108
109   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
110   void runAtExits(void *DSOHandle);
111
112   /// Returns the base address of the section containing ThreadData.
113   Expected<std::pair<const char *, size_t>>
114   getThreadDataSectionFor(const char *ThreadData);
115
116   void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
117
118 private:
119   PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
120   PerJITDylibState *getJITDylibStateByName(string_view Path);
121   PerJITDylibState &
122   getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
123
124   Error registerThreadDataSection(span<const char> ThreadDataSection);
125
126   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
127                                                 string_view Symbol);
128
129   Expected<ELFNixJITDylibInitializerSequence>
130   getJITDylibInitializersByName(string_view Path);
131   Expected<void *> dlopenInitialize(string_view Path, int Mode);
132   Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
133
134   static ELFNixPlatformRuntimeState *MOPS;
135
136   using InitSectionHandler =
137       Error (*)(const std::vector<ExecutorAddrRange> &Sections,
138                 const ELFNixJITDylibInitializers &MOJDIs);
139   const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
140       {{".init_array", runInitArray}};
141
142   void *PlatformJDDSOHandle;
143
144   // FIXME: Move to thread-state.
145   std::string DLFcnError;
146
147   std::recursive_mutex JDStatesMutex;
148   std::unordered_map<void *, PerJITDylibState> JDStates;
149   std::unordered_map<std::string, void *> JDNameToHeader;
150
151   std::mutex ThreadDataSectionsMutex;
152   std::map<const char *, size_t> ThreadDataSections;
153 };
154
155 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
156
157 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
158   assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
159   MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
160 }
161
162 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
163   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
164   return *MOPS;
165 }
166
167 void ELFNixPlatformRuntimeState::destroy() {
168   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
169   delete MOPS;
170 }
171
172 Error ELFNixPlatformRuntimeState::registerObjectSections(
173     ELFNixPerObjectSectionsToRegister POSR) {
174   if (POSR.EHFrameSection.Start)
175     __register_frame(POSR.EHFrameSection.Start.toPtr<const char *>());
176
177   if (POSR.ThreadDataSection.Start) {
178     if (auto Err = registerThreadDataSection(
179             POSR.ThreadDataSection.toSpan<const char>()))
180       return Err;
181   }
182
183   return Error::success();
184 }
185
186 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
187     ELFNixPerObjectSectionsToRegister POSR) {
188   if (POSR.EHFrameSection.Start)
189     __deregister_frame(POSR.EHFrameSection.Start.toPtr<const char *>());
190
191   return Error::success();
192 }
193
194 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
195
196 void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
197   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
198
199   // Use fast path if all JITDylibs are already loaded and don't require
200   // re-running initializers.
201   if (auto *JDS = getJITDylibStateByName(Path)) {
202     if (!JDS->AllowReinitialization) {
203       ++JDS->RefCount;
204       return JDS->Header;
205     }
206   }
207
208   auto H = dlopenInitialize(Path, Mode);
209   if (!H) {
210     DLFcnError = toString(H.takeError());
211     return nullptr;
212   }
213
214   return *H;
215 }
216
217 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
218   runAtExits(DSOHandle);
219   return 0;
220 }
221
222 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
223   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
224   if (!Addr) {
225     DLFcnError = toString(Addr.takeError());
226     return 0;
227   }
228
229   return Addr->toPtr<void *>();
230 }
231
232 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
233                                                void *DSOHandle) {
234   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
235   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
236   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
237   assert(JDS && "JITDylib state not initialized");
238   JDS->AtExits.push_back({F, Arg});
239   return 0;
240 }
241
242 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
243   // FIXME: Should atexits be allowed to run concurrently with access to
244   // JDState?
245   AtExitsVector V;
246   {
247     std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
248     auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
249     assert(JDS && "JITDlybi state not initialized");
250     std::swap(V, JDS->AtExits);
251   }
252
253   while (!V.empty()) {
254     auto &AE = V.back();
255     AE.Func(AE.Arg);
256     V.pop_back();
257   }
258 }
259
260 Expected<std::pair<const char *, size_t>>
261 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
262   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
263   auto I = ThreadDataSections.upper_bound(ThreadData);
264   // Check that we have a valid entry conovering this address.
265   if (I == ThreadDataSections.begin())
266     return make_error<StringError>("No thread local data section for key");
267   I = std::prev(I);
268   if (ThreadData >= I->first + I->second)
269     return make_error<StringError>("No thread local data section for key");
270   return *I;
271 }
272
273 ELFNixPlatformRuntimeState::PerJITDylibState *
274 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
275   auto I = JDStates.find(DSOHandle);
276   if (I == JDStates.end())
277     return nullptr;
278   return &I->second;
279 }
280
281 ELFNixPlatformRuntimeState::PerJITDylibState *
282 ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
283   // FIXME: Avoid creating string copy here.
284   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
285   if (I == JDNameToHeader.end())
286     return nullptr;
287   void *H = I->second;
288   auto J = JDStates.find(H);
289   assert(J != JDStates.end() &&
290          "JITDylib has name map entry but no header map entry");
291   return &J->second;
292 }
293
294 ELFNixPlatformRuntimeState::PerJITDylibState &
295 ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
296     ELFNixJITDylibInitializers &MOJDIs) {
297   void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
298
299   auto &JDS = JDStates[Header];
300
301   // If this entry hasn't been created yet.
302   if (!JDS.Header) {
303     assert(!JDNameToHeader.count(MOJDIs.Name) &&
304            "JITDylib has header map entry but no name map entry");
305     JDNameToHeader[MOJDIs.Name] = Header;
306     JDS.Header = Header;
307   }
308
309   return JDS;
310 }
311
312 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
313     span<const char> ThreadDataSection) {
314   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
315   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
316   if (I != ThreadDataSections.begin()) {
317     auto J = std::prev(I);
318     if (J->first + J->second > ThreadDataSection.data())
319       return make_error<StringError>("Overlapping .tdata sections");
320   }
321   ThreadDataSections.insert(
322       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
323   return Error::success();
324 }
325
326 Expected<ExecutorAddr>
327 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
328                                                    string_view Sym) {
329   Expected<ExecutorAddr> Result((ExecutorAddr()));
330   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
331           SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
332                                              Result,
333                                              ExecutorAddr::fromPtr(DSOHandle),
334                                              Sym))
335     return std::move(Err);
336   return Result;
337 }
338
339 Expected<ELFNixJITDylibInitializerSequence>
340 ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
341   Expected<ELFNixJITDylibInitializerSequence> Result(
342       (ELFNixJITDylibInitializerSequence()));
343   std::string PathStr(Path.data(), Path.size());
344   if (auto Err =
345           WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
346               SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
347                                 Path))
348     return std::move(Err);
349   return Result;
350 }
351
352 Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
353                                                               int Mode) {
354   // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
355   // reinitialization. We need to call in to the JIT to see if there's any new
356   // work pending.
357   auto InitSeq = getJITDylibInitializersByName(Path);
358   if (!InitSeq)
359     return InitSeq.takeError();
360
361   // Init sequences should be non-empty.
362   if (InitSeq->empty())
363     return make_error<StringError>(
364         "__orc_rt_elfnix_get_initializers returned an "
365         "empty init sequence");
366
367   // Otherwise register and run initializers for each JITDylib.
368   for (auto &MOJDIs : *InitSeq)
369     if (auto Err = initializeJITDylib(MOJDIs))
370       return std::move(Err);
371
372   // Return the header for the last item in the list.
373   auto *JDS = getJITDylibStateByHeaderAddr(
374       InitSeq->back().DSOHandleAddress.toPtr<void *>());
375   assert(JDS && "Missing state entry for JD");
376   return JDS->Header;
377 }
378
379 Error ELFNixPlatformRuntimeState::initializeJITDylib(
380     ELFNixJITDylibInitializers &MOJDIs) {
381
382   auto &JDS = getOrCreateJITDylibState(MOJDIs);
383   ++JDS.RefCount;
384
385   for (auto &KV : InitSections) {
386     const auto &Name = KV.first;
387     const auto &Handler = KV.second;
388     auto I = MOJDIs.InitSections.find(Name);
389     if (I != MOJDIs.InitSections.end()) {
390       if (auto Err = Handler(I->second, MOJDIs))
391         return Err;
392     }
393   }
394
395   return Error::success();
396 }
397 class ELFNixPlatformRuntimeTLVManager {
398 public:
399   void *getInstance(const char *ThreadData);
400
401 private:
402   std::unordered_map<const char *, char *> Instances;
403   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
404 };
405
406 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
407   auto I = Instances.find(ThreadData);
408   if (I != Instances.end())
409     return I->second;
410   auto TDS =
411       ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
412   if (!TDS) {
413     __orc_rt_log_error(toString(TDS.takeError()).c_str());
414     return nullptr;
415   }
416
417   auto &Allocated = AllocatedSections[TDS->first];
418   if (!Allocated) {
419     Allocated = std::make_unique<char[]>(TDS->second);
420     memcpy(Allocated.get(), TDS->first, TDS->second);
421   }
422   size_t ThreadDataDelta = ThreadData - TDS->first;
423   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
424
425   char *Instance = Allocated.get() + ThreadDataDelta;
426   Instances[ThreadData] = Instance;
427   return Instance;
428 }
429
430 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
431   delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
432 }
433
434 } // end anonymous namespace
435
436 //------------------------------------------------------------------------------
437 //                             JIT entry points
438 //------------------------------------------------------------------------------
439
440 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
441 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
442   return WrapperFunction<void(uint64_t)>::handle(
443              ArgData, ArgSize,
444              [](uint64_t &DSOHandle) {
445                ELFNixPlatformRuntimeState::initialize(
446                    reinterpret_cast<void *>(DSOHandle));
447              })
448       .release();
449 }
450
451 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
452 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
453   ELFNixPlatformRuntimeState::destroy();
454   return WrapperFunctionResult().release();
455 }
456
457 /// Wrapper function for registering metadata on a per-object basis.
458 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
459 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
460   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
461       handle(ArgData, ArgSize,
462              [](ELFNixPerObjectSectionsToRegister &POSR) {
463                return ELFNixPlatformRuntimeState::get().registerObjectSections(
464                    std::move(POSR));
465              })
466           .release();
467 }
468
469 /// Wrapper for releasing per-object metadat.
470 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
471 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
472   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
473       handle(ArgData, ArgSize,
474              [](ELFNixPerObjectSectionsToRegister &POSR) {
475                return ELFNixPlatformRuntimeState::get()
476                    .deregisterObjectSections(std::move(POSR));
477              })
478           .release();
479 }
480
481 //------------------------------------------------------------------------------
482 //                           TLV support
483 //------------------------------------------------------------------------------
484
485 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
486   auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
487       pthread_getspecific(D->Key));
488   if (!TLVMgr)
489     TLVMgr = new ELFNixPlatformRuntimeTLVManager();
490   if (pthread_setspecific(D->Key, TLVMgr)) {
491     __orc_rt_log_error("Call to pthread_setspecific failed");
492     return nullptr;
493   }
494
495   return TLVMgr->getInstance(
496       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
497 }
498
499 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
500 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
501   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
502              ArgData, ArgSize,
503              []() -> Expected<uint64_t> {
504                pthread_key_t Key;
505                if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
506                  __orc_rt_log_error("Call to pthread_key_create failed");
507                  return make_error<StringError>(strerror(Err));
508                }
509                return static_cast<uint64_t>(Key);
510              })
511       .release();
512 }
513
514 //------------------------------------------------------------------------------
515 //                           cxa_atexit support
516 //------------------------------------------------------------------------------
517
518 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
519                                void *dso_handle) {
520   return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
521                                                           dso_handle);
522 }
523
524 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
525   auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
526   return ELFNixPlatformRuntimeState::get().registerAtExit(
527       func, NULL, PlatformRTState.getPlatformJDDSOHandle());
528 }
529
530 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
531   ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
532 }
533
534 //------------------------------------------------------------------------------
535 //                        JIT'd dlfcn alternatives.
536 //------------------------------------------------------------------------------
537
538 const char *__orc_rt_elfnix_jit_dlerror() {
539   return ELFNixPlatformRuntimeState::get().dlerror();
540 }
541
542 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
543   return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
544 }
545
546 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
547   return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
548 }
549
550 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
551   return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
552 }
553
554 //------------------------------------------------------------------------------
555 //                             ELFNix Run Program
556 //------------------------------------------------------------------------------
557
558 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
559     const char *JITDylibName, const char *EntrySymbolName, int argc,
560     char *argv[]) {
561   using MainTy = int (*)(int, char *[]);
562
563   void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
564                                        __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
565   if (!H) {
566     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
567     return -1;
568   }
569
570   auto *Main =
571       reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
572
573   if (!Main) {
574     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
575     return -1;
576   }
577
578   int Result = Main(argc, argv);
579
580   if (__orc_rt_elfnix_jit_dlclose(H) == -1)
581     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
582
583   return Result;
584 }