1 //===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of XRay, a dynamic runtime instrumentation system.
11 // Implementation of the API functions.
13 //===----------------------------------------------------------------------===//
15 #include "xray_interface_internal.h"
25 #include <zircon/process.h>
26 #include <zircon/sanitizer.h>
27 #include <zircon/status.h>
28 #include <zircon/syscalls.h>
31 #include "sanitizer_common/sanitizer_addrhashmap.h"
32 #include "sanitizer_common/sanitizer_common.h"
34 #include "xray_defs.h"
35 #include "xray_flags.h"
37 extern __sanitizer::SpinMutex XRayInstrMapMutex;
38 extern __sanitizer::atomic_uint8_t XRayInitialized;
39 extern __xray::XRaySledMap XRayInstrMap;
43 #if defined(__x86_64__)
44 static const int16_t cSledLength = 12;
45 #elif defined(__aarch64__)
46 static const int16_t cSledLength = 32;
47 #elif defined(__arm__)
48 static const int16_t cSledLength = 28;
49 #elif SANITIZER_MIPS32
50 static const int16_t cSledLength = 48;
51 #elif SANITIZER_MIPS64
52 static const int16_t cSledLength = 64;
53 #elif defined(__powerpc64__)
54 static const int16_t cSledLength = 8;
56 #error "Unsupported CPU Architecture"
57 #endif /* CPU architecture */
59 // This is the function to call when we encounter the entry or exit sleds.
60 atomic_uintptr_t XRayPatchedFunction{0};
62 // This is the function to call from the arg1-enabled sleds/trampolines.
63 atomic_uintptr_t XRayArgLogger{0};
65 // This is the function to call when we encounter a custom event log call.
66 atomic_uintptr_t XRayPatchedCustomEvent{0};
68 // This is the function to call when we encounter a typed event log call.
69 atomic_uintptr_t XRayPatchedTypedEvent{0};
71 // This is the global status to determine whether we are currently
72 // patching/unpatching.
73 atomic_uint8_t XRayPatching{0};
75 struct TypeDescription {
77 std::size_t description_string_length;
80 using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
81 // An address map from immutable descriptors to type ids.
82 TypeDescriptorMapType TypeDescriptorAddressMap{};
84 atomic_uint32_t TypeEventDescriptorCounter{0};
86 // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
87 // undo any successful mprotect(...) changes. This is used to make a page
88 // writeable and executable, and upon destruction if it was successful in
89 // doing so returns the page into a read-only and executable page.
91 // This is only used specifically for runtime-patching of the XRay
92 // instrumentation points. This assumes that the executable pages are
93 // originally read-and-execute only.
94 class MProtectHelper {
95 void *PageAlignedAddr;
96 std::size_t MProtectLen;
100 explicit MProtectHelper(void *PageAlignedAddr,
101 std::size_t MProtectLen,
102 std::size_t PageSize) XRAY_NEVER_INSTRUMENT
103 : PageAlignedAddr(PageAlignedAddr),
104 MProtectLen(MProtectLen),
106 #if SANITIZER_FUCHSIA
107 MProtectLen = RoundUpTo(MProtectLen, PageSize);
111 int MakeWriteable() XRAY_NEVER_INSTRUMENT {
112 #if SANITIZER_FUCHSIA
113 auto R = __sanitizer_change_code_protection(
114 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
116 Report("XRay: cannot change code protection: %s\n",
117 _zx_status_get_string(R));
123 auto R = mprotect(PageAlignedAddr, MProtectLen,
124 PROT_READ | PROT_WRITE | PROT_EXEC);
131 ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
133 #if SANITIZER_FUCHSIA
134 auto R = __sanitizer_change_code_protection(
135 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
137 Report("XRay: cannot change code protection: %s\n",
138 _zx_status_get_string(R));
141 mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
149 bool patchSled(const XRaySledEntry &Sled, bool Enable,
150 int32_t FuncId) XRAY_NEVER_INSTRUMENT {
151 bool Success = false;
153 case XRayEntryType::ENTRY:
154 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
156 case XRayEntryType::EXIT:
157 Success = patchFunctionExit(Enable, FuncId, Sled);
159 case XRayEntryType::TAIL:
160 Success = patchFunctionTailExit(Enable, FuncId, Sled);
162 case XRayEntryType::LOG_ARGS_ENTRY:
163 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
165 case XRayEntryType::CUSTOM_EVENT:
166 Success = patchCustomEvent(Enable, FuncId, Sled);
168 case XRayEntryType::TYPED_EVENT:
169 Success = patchTypedEvent(Enable, FuncId, Sled);
172 Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
178 const XRayFunctionSledIndex
179 findFunctionSleds(int32_t FuncId,
180 const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
182 uint64_t LastFnAddr = 0;
183 XRayFunctionSledIndex Index = {nullptr, nullptr};
185 for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
186 const auto &Sled = InstrMap.Sleds[I];
187 const auto Function = Sled.function();
188 if (Function != LastFnAddr) {
190 LastFnAddr = Function;
193 if (CurFn == FuncId) {
194 if (Index.Begin == nullptr)
205 XRayPatchingStatus patchFunction(int32_t FuncId,
206 bool Enable) XRAY_NEVER_INSTRUMENT {
207 if (!atomic_load(&XRayInitialized,
208 memory_order_acquire))
209 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
211 uint8_t NotPatching = false;
212 if (!atomic_compare_exchange_strong(
213 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
214 return XRayPatchingStatus::ONGOING; // Already patching.
216 // Next, we look for the function index.
217 XRaySledMap InstrMap;
219 SpinMutexLock Guard(&XRayInstrMapMutex);
220 InstrMap = XRayInstrMap;
223 // If we don't have an index, we can't patch individual functions.
224 if (InstrMap.Functions == 0)
225 return XRayPatchingStatus::NOT_INITIALIZED;
227 // FuncId must be a positive number, less than the number of functions
229 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
230 Report("Invalid function id provided: %d\n", FuncId);
231 return XRayPatchingStatus::FAILED;
234 // Now we patch ths sleds for this specific function.
235 auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
236 : findFunctionSleds(FuncId, InstrMap);
237 auto *f = SledRange.Begin;
238 auto *e = SledRange.End;
239 bool SucceedOnce = false;
241 SucceedOnce |= patchSled(*f++, Enable, FuncId);
243 atomic_store(&XRayPatching, false,
244 memory_order_release);
247 Report("Failed patching any sled for function '%d'.", FuncId);
248 return XRayPatchingStatus::FAILED;
251 return XRayPatchingStatus::SUCCESS;
254 // controlPatching implements the common internals of the patching/unpatching
255 // implementation. |Enable| defines whether we're enabling or disabling the
256 // runtime XRay instrumentation.
257 XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
258 if (!atomic_load(&XRayInitialized,
259 memory_order_acquire))
260 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
262 uint8_t NotPatching = false;
263 if (!atomic_compare_exchange_strong(
264 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
265 return XRayPatchingStatus::ONGOING; // Already patching.
267 uint8_t PatchingSuccess = false;
268 auto XRayPatchingStatusResetter =
269 at_scope_exit([&PatchingSuccess] {
270 if (!PatchingSuccess)
271 atomic_store(&XRayPatching, false,
272 memory_order_release);
275 XRaySledMap InstrMap;
277 SpinMutexLock Guard(&XRayInstrMapMutex);
278 InstrMap = XRayInstrMap;
280 if (InstrMap.Entries == 0)
281 return XRayPatchingStatus::NOT_INITIALIZED;
286 // First we want to find the bounds for which we have instrumentation points,
287 // and try to get as few calls to mprotect(...) as possible. We're assuming
288 // that all the sleds for the instrumentation map are contiguous as a single
289 // set of pages. When we do support dynamic shared object instrumentation,
290 // we'll need to do this for each set of page load offsets per DSO loaded. For
291 // now we're assuming we can mprotect the whole section of text between the
292 // minimum sled address and the maximum sled address (+ the largest sled
294 auto *MinSled = &InstrMap.Sleds[0];
295 auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
296 for (std::size_t I = 0; I < InstrMap.Entries; I++) {
297 const auto &Sled = InstrMap.Sleds[I];
298 if (Sled.address() < MinSled->address())
300 if (Sled.address() > MaxSled->address())
304 const size_t PageSize = flags()->xray_page_size_override > 0
305 ? flags()->xray_page_size_override
306 : GetPageSizeCached();
307 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
308 Report("System page size is not a power of two: %lld\n", PageSize);
309 return XRayPatchingStatus::FAILED;
312 void *PageAlignedAddr =
313 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
315 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
317 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
318 if (Protector.MakeWriteable() == -1) {
319 Report("Failed mprotect: %d\n", errno);
320 return XRayPatchingStatus::FAILED;
323 for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
324 auto &Sled = InstrMap.Sleds[I];
325 auto F = Sled.function();
332 patchSled(Sled, Enable, FuncId);
334 atomic_store(&XRayPatching, false,
335 memory_order_release);
336 PatchingSuccess = true;
337 return XRayPatchingStatus::SUCCESS;
340 XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
341 bool Enable) XRAY_NEVER_INSTRUMENT {
342 XRaySledMap InstrMap;
344 SpinMutexLock Guard(&XRayInstrMapMutex);
345 InstrMap = XRayInstrMap;
348 // FuncId must be a positive number, less than the number of functions
350 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
351 Report("Invalid function id provided: %d\n", FuncId);
352 return XRayPatchingStatus::FAILED;
355 const size_t PageSize = flags()->xray_page_size_override > 0
356 ? flags()->xray_page_size_override
357 : GetPageSizeCached();
358 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
359 Report("Provided page size is not a power of two: %lld\n", PageSize);
360 return XRayPatchingStatus::FAILED;
363 // Here we compute the minumum sled and maximum sled associated with a
364 // particular function ID.
365 auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
366 : findFunctionSleds(FuncId, InstrMap);
367 auto *f = SledRange.Begin;
368 auto *e = SledRange.End;
370 auto *MaxSled = (SledRange.End - 1);
372 if (f->address() < MinSled->address())
374 if (f->address() > MaxSled->address())
379 void *PageAlignedAddr =
380 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
382 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
384 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
385 if (Protector.MakeWriteable() == -1) {
386 Report("Failed mprotect: %d\n", errno);
387 return XRayPatchingStatus::FAILED;
389 return patchFunction(FuncId, Enable);
394 } // namespace __xray
396 using namespace __xray;
398 // The following functions are declared `extern "C" {...}` in the header, hence
399 // they're defined in the global namespace.
401 int __xray_set_handler(void (*entry)(int32_t,
402 XRayEntryType)) XRAY_NEVER_INSTRUMENT {
403 if (atomic_load(&XRayInitialized,
404 memory_order_acquire)) {
406 atomic_store(&__xray::XRayPatchedFunction,
407 reinterpret_cast<uintptr_t>(entry),
408 memory_order_release);
414 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
415 XRAY_NEVER_INSTRUMENT {
416 if (atomic_load(&XRayInitialized,
417 memory_order_acquire)) {
418 atomic_store(&__xray::XRayPatchedCustomEvent,
419 reinterpret_cast<uintptr_t>(entry),
420 memory_order_release);
426 int __xray_set_typedevent_handler(void (*entry)(
427 uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
428 if (atomic_load(&XRayInitialized,
429 memory_order_acquire)) {
430 atomic_store(&__xray::XRayPatchedTypedEvent,
431 reinterpret_cast<uintptr_t>(entry),
432 memory_order_release);
438 int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
439 return __xray_set_handler(nullptr);
442 int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
443 return __xray_set_customevent_handler(nullptr);
446 int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
447 return __xray_set_typedevent_handler(nullptr);
450 uint16_t __xray_register_event_type(
451 const char *const event_type) XRAY_NEVER_INSTRUMENT {
452 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
454 h->type_id = atomic_fetch_add(
455 &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
456 h->description_string_length = strnlen(event_type, 1024);
461 XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
462 return controlPatching(true);
465 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
466 return controlPatching(false);
469 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
470 return mprotectAndPatchFunction(FuncId, true);
474 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
475 return mprotectAndPatchFunction(FuncId, false);
478 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
479 if (!atomic_load(&XRayInitialized,
480 memory_order_acquire))
483 // A relaxed write might not be visible even if the current thread gets
484 // scheduled on a different CPU/NUMA node. We need to wait for everyone to
485 // have this handler installed for consistency of collected data across CPUs.
486 atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
487 memory_order_release);
491 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
493 uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
494 XRaySledMap InstrMap;
496 SpinMutexLock Guard(&XRayInstrMapMutex);
497 InstrMap = XRayInstrMap;
500 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
502 const XRaySledEntry *Sled = InstrMap.SledsIndex
503 ? InstrMap.SledsIndex[FuncId - 1].Begin
504 : findFunctionSleds(FuncId, InstrMap).Begin;
505 return Sled->function()
506 // On PPC, function entries are always aligned to 16 bytes. The beginning of a
507 // sled might be a local entry, which is always +8 based on the global entry.
508 // Always return the global entry.
515 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
516 SpinMutexLock Guard(&XRayInstrMapMutex);
517 return XRayInstrMap.Functions;