1 //===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of XRay, a dynamic runtime instrumentation system.
11 // Implementation of the API functions.
13 //===----------------------------------------------------------------------===//
15 #include "xray_interface_internal.h"
25 #include <zircon/process.h>
26 #include <zircon/sanitizer.h>
27 #include <zircon/status.h>
28 #include <zircon/syscalls.h>
31 #include "sanitizer_common/sanitizer_addrhashmap.h"
32 #include "sanitizer_common/sanitizer_common.h"
34 #include "xray_defs.h"
35 #include "xray_flags.h"
37 extern __sanitizer::SpinMutex XRayInstrMapMutex;
38 extern __sanitizer::atomic_uint8_t XRayInitialized;
39 extern __xray::XRaySledMap XRayInstrMap;
43 #if defined(__x86_64__)
44 static const int16_t cSledLength = 12;
45 #elif defined(__aarch64__)
46 static const int16_t cSledLength = 32;
47 #elif defined(__arm__)
48 static const int16_t cSledLength = 28;
49 #elif SANITIZER_MIPS32
50 static const int16_t cSledLength = 48;
51 #elif SANITIZER_MIPS64
52 static const int16_t cSledLength = 64;
53 #elif defined(__powerpc64__)
54 static const int16_t cSledLength = 8;
56 #error "Unsupported CPU Architecture"
57 #endif /* CPU architecture */
59 // This is the function to call when we encounter the entry or exit sleds.
60 atomic_uintptr_t XRayPatchedFunction{0};
62 // This is the function to call from the arg1-enabled sleds/trampolines.
63 atomic_uintptr_t XRayArgLogger{0};
65 // This is the function to call when we encounter a custom event log call.
66 atomic_uintptr_t XRayPatchedCustomEvent{0};
68 // This is the function to call when we encounter a typed event log call.
69 atomic_uintptr_t XRayPatchedTypedEvent{0};
71 // This is the global status to determine whether we are currently
72 // patching/unpatching.
73 atomic_uint8_t XRayPatching{0};
75 struct TypeDescription {
77 std::size_t description_string_length;
80 using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
81 // An address map from immutable descriptors to type ids.
82 TypeDescriptorMapType TypeDescriptorAddressMap{};
84 atomic_uint32_t TypeEventDescriptorCounter{0};
86 // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
87 // undo any successful mprotect(...) changes. This is used to make a page
88 // writeable and executable, and upon destruction if it was successful in
89 // doing so returns the page into a read-only and executable page.
91 // This is only used specifically for runtime-patching of the XRay
92 // instrumentation points. This assumes that the executable pages are
93 // originally read-and-execute only.
94 class MProtectHelper {
95 void *PageAlignedAddr;
96 std::size_t MProtectLen;
100 explicit MProtectHelper(void *PageAlignedAddr,
101 std::size_t MProtectLen,
102 std::size_t PageSize) XRAY_NEVER_INSTRUMENT
103 : PageAlignedAddr(PageAlignedAddr),
104 MProtectLen(MProtectLen),
106 #if SANITIZER_FUCHSIA
107 MProtectLen = RoundUpTo(MProtectLen, PageSize);
111 int MakeWriteable() XRAY_NEVER_INSTRUMENT {
112 #if SANITIZER_FUCHSIA
113 auto R = __sanitizer_change_code_protection(
114 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
116 Report("XRay: cannot change code protection: %s\n",
117 _zx_status_get_string(R));
123 auto R = mprotect(PageAlignedAddr, MProtectLen,
124 PROT_READ | PROT_WRITE | PROT_EXEC);
131 ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
133 #if SANITIZER_FUCHSIA
134 auto R = __sanitizer_change_code_protection(
135 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
137 Report("XRay: cannot change code protection: %s\n",
138 _zx_status_get_string(R));
141 mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
149 bool patchSled(const XRaySledEntry &Sled, bool Enable,
150 int32_t FuncId) XRAY_NEVER_INSTRUMENT {
151 bool Success = false;
153 case XRayEntryType::ENTRY:
154 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
156 case XRayEntryType::EXIT:
157 Success = patchFunctionExit(Enable, FuncId, Sled);
159 case XRayEntryType::TAIL:
160 Success = patchFunctionTailExit(Enable, FuncId, Sled);
162 case XRayEntryType::LOG_ARGS_ENTRY:
163 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
165 case XRayEntryType::CUSTOM_EVENT:
166 Success = patchCustomEvent(Enable, FuncId, Sled);
168 case XRayEntryType::TYPED_EVENT:
169 Success = patchTypedEvent(Enable, FuncId, Sled);
172 Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
178 XRayPatchingStatus patchFunction(int32_t FuncId,
179 bool Enable) XRAY_NEVER_INSTRUMENT {
180 if (!atomic_load(&XRayInitialized,
181 memory_order_acquire))
182 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
184 uint8_t NotPatching = false;
185 if (!atomic_compare_exchange_strong(
186 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
187 return XRayPatchingStatus::ONGOING; // Already patching.
189 // Next, we look for the function index.
190 XRaySledMap InstrMap;
192 SpinMutexLock Guard(&XRayInstrMapMutex);
193 InstrMap = XRayInstrMap;
196 // If we don't have an index, we can't patch individual functions.
197 if (InstrMap.Functions == 0)
198 return XRayPatchingStatus::NOT_INITIALIZED;
200 // FuncId must be a positive number, less than the number of functions
202 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
203 Report("Invalid function id provided: %d\n", FuncId);
204 return XRayPatchingStatus::FAILED;
207 // Now we patch ths sleds for this specific function.
208 auto SledRange = InstrMap.SledsIndex[FuncId - 1];
209 auto *f = SledRange.Begin;
210 auto *e = SledRange.End;
212 bool SucceedOnce = false;
214 SucceedOnce |= patchSled(*f++, Enable, FuncId);
216 atomic_store(&XRayPatching, false,
217 memory_order_release);
220 Report("Failed patching any sled for function '%d'.", FuncId);
221 return XRayPatchingStatus::FAILED;
224 return XRayPatchingStatus::SUCCESS;
227 // controlPatching implements the common internals of the patching/unpatching
228 // implementation. |Enable| defines whether we're enabling or disabling the
229 // runtime XRay instrumentation.
230 XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
231 if (!atomic_load(&XRayInitialized,
232 memory_order_acquire))
233 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
235 uint8_t NotPatching = false;
236 if (!atomic_compare_exchange_strong(
237 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
238 return XRayPatchingStatus::ONGOING; // Already patching.
240 uint8_t PatchingSuccess = false;
241 auto XRayPatchingStatusResetter =
242 at_scope_exit([&PatchingSuccess] {
243 if (!PatchingSuccess)
244 atomic_store(&XRayPatching, false,
245 memory_order_release);
248 XRaySledMap InstrMap;
250 SpinMutexLock Guard(&XRayInstrMapMutex);
251 InstrMap = XRayInstrMap;
253 if (InstrMap.Entries == 0)
254 return XRayPatchingStatus::NOT_INITIALIZED;
259 // First we want to find the bounds for which we have instrumentation points,
260 // and try to get as few calls to mprotect(...) as possible. We're assuming
261 // that all the sleds for the instrumentation map are contiguous as a single
262 // set of pages. When we do support dynamic shared object instrumentation,
263 // we'll need to do this for each set of page load offsets per DSO loaded. For
264 // now we're assuming we can mprotect the whole section of text between the
265 // minimum sled address and the maximum sled address (+ the largest sled
267 auto MinSled = InstrMap.Sleds[0];
268 auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1];
269 for (std::size_t I = 0; I < InstrMap.Entries; I++) {
270 const auto &Sled = InstrMap.Sleds[I];
271 if (Sled.Address < MinSled.Address)
273 if (Sled.Address > MaxSled.Address)
277 const size_t PageSize = flags()->xray_page_size_override > 0
278 ? flags()->xray_page_size_override
279 : GetPageSizeCached();
280 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
281 Report("System page size is not a power of two: %lld\n", PageSize);
282 return XRayPatchingStatus::FAILED;
285 void *PageAlignedAddr =
286 reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
288 (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
289 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
290 if (Protector.MakeWriteable() == -1) {
291 Report("Failed mprotect: %d\n", errno);
292 return XRayPatchingStatus::FAILED;
295 for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
296 auto &Sled = InstrMap.Sleds[I];
297 auto F = Sled.Function;
304 patchSled(Sled, Enable, FuncId);
306 atomic_store(&XRayPatching, false,
307 memory_order_release);
308 PatchingSuccess = true;
309 return XRayPatchingStatus::SUCCESS;
312 XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
313 bool Enable) XRAY_NEVER_INSTRUMENT {
314 XRaySledMap InstrMap;
316 SpinMutexLock Guard(&XRayInstrMapMutex);
317 InstrMap = XRayInstrMap;
320 // FuncId must be a positive number, less than the number of functions
322 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
323 Report("Invalid function id provided: %d\n", FuncId);
324 return XRayPatchingStatus::FAILED;
327 const size_t PageSize = flags()->xray_page_size_override > 0
328 ? flags()->xray_page_size_override
329 : GetPageSizeCached();
330 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
331 Report("Provided page size is not a power of two: %lld\n", PageSize);
332 return XRayPatchingStatus::FAILED;
335 // Here we compute the minumum sled and maximum sled associated with a
336 // particular function ID.
337 auto SledRange = InstrMap.SledsIndex[FuncId - 1];
338 auto *f = SledRange.Begin;
339 auto *e = SledRange.End;
341 auto MaxSled = *(SledRange.End - 1);
343 if (f->Address < MinSled.Address)
345 if (f->Address > MaxSled.Address)
350 void *PageAlignedAddr =
351 reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
353 (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
354 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
355 if (Protector.MakeWriteable() == -1) {
356 Report("Failed mprotect: %d\n", errno);
357 return XRayPatchingStatus::FAILED;
359 return patchFunction(FuncId, Enable);
364 } // namespace __xray
366 using namespace __xray;
368 // The following functions are declared `extern "C" {...}` in the header, hence
369 // they're defined in the global namespace.
371 int __xray_set_handler(void (*entry)(int32_t,
372 XRayEntryType)) XRAY_NEVER_INSTRUMENT {
373 if (atomic_load(&XRayInitialized,
374 memory_order_acquire)) {
376 atomic_store(&__xray::XRayPatchedFunction,
377 reinterpret_cast<uintptr_t>(entry),
378 memory_order_release);
384 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
385 XRAY_NEVER_INSTRUMENT {
386 if (atomic_load(&XRayInitialized,
387 memory_order_acquire)) {
388 atomic_store(&__xray::XRayPatchedCustomEvent,
389 reinterpret_cast<uintptr_t>(entry),
390 memory_order_release);
396 int __xray_set_typedevent_handler(void (*entry)(
397 uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
398 if (atomic_load(&XRayInitialized,
399 memory_order_acquire)) {
400 atomic_store(&__xray::XRayPatchedTypedEvent,
401 reinterpret_cast<uintptr_t>(entry),
402 memory_order_release);
408 int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
409 return __xray_set_handler(nullptr);
412 int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
413 return __xray_set_customevent_handler(nullptr);
416 int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
417 return __xray_set_typedevent_handler(nullptr);
420 uint16_t __xray_register_event_type(
421 const char *const event_type) XRAY_NEVER_INSTRUMENT {
422 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
424 h->type_id = atomic_fetch_add(
425 &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
426 h->description_string_length = strnlen(event_type, 1024);
431 XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
432 return controlPatching(true);
435 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
436 return controlPatching(false);
439 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
440 return mprotectAndPatchFunction(FuncId, true);
444 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
445 return mprotectAndPatchFunction(FuncId, false);
448 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
449 if (!atomic_load(&XRayInitialized,
450 memory_order_acquire))
453 // A relaxed write might not be visible even if the current thread gets
454 // scheduled on a different CPU/NUMA node. We need to wait for everyone to
455 // have this handler installed for consistency of collected data across CPUs.
456 atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
457 memory_order_release);
461 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
463 uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
464 SpinMutexLock Guard(&XRayInstrMapMutex);
465 if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions)
467 return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function
468 // On PPC, function entries are always aligned to 16 bytes. The beginning of a
469 // sled might be a local entry, which is always +8 based on the global entry.
470 // Always return the global entry.
477 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
478 SpinMutexLock Guard(&XRayInstrMapMutex);
479 return XRayInstrMap.Functions;