1 //===-- xray_allocator.h ---------------------------------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of XRay, a dynamic runtime instrumentation system.
12 // Defines the allocator interface for an arena allocator, used primarily for
13 // the profiling runtime.
15 //===----------------------------------------------------------------------===//
16 #ifndef XRAY_ALLOCATOR_H
17 #define XRAY_ALLOCATOR_H
19 #include "sanitizer_common/sanitizer_common.h"
20 #include "sanitizer_common/sanitizer_internal_defs.h"
21 #include "sanitizer_common/sanitizer_mutex.h"
23 #include <zircon/process.h>
24 #include <zircon/status.h>
25 #include <zircon/syscalls.h>
27 #include "sanitizer_common/sanitizer_posix.h"
29 #include "xray_defs.h"
30 #include "xray_utils.h"
37 // We implement our own memory allocation routine which will bypass the
38 // internal allocator. This allows us to manage the memory directly, using
39 // mmap'ed memory to back the allocators.
40 template <class T> T *allocate() XRAY_NEVER_INSTRUMENT {
41 uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
44 zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo);
45 if (Status != ZX_OK) {
47 Report("XRay Profiling: Failed to create VMO of size %zu: %s\n",
48 sizeof(T), _zx_status_get_string(Status));
53 _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
54 Vmo, 0, sizeof(T), &B);
55 _zx_handle_close(Vmo);
56 if (Status != ZX_OK) {
58 Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", sizeof(T),
59 _zx_status_get_string(Status));
62 return reinterpret_cast<T *>(B);
64 uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
65 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
67 if (UNLIKELY(internal_iserror(B, &ErrNo))) {
70 "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
75 return reinterpret_cast<T *>(B);
78 template <class T> void deallocate(T *B) XRAY_NEVER_INSTRUMENT {
81 uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
83 _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B),
86 internal_munmap(B, RoundedSize);
90 template <class T = unsigned char>
91 T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
92 uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
95 zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo);
96 if (Status != ZX_OK) {
98 Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", S,
99 _zx_status_get_string(Status));
103 Status = _zx_vmar_map(_zx_vmar_root_self(),
104 ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, Vmo, 0, S, &B);
105 _zx_handle_close(Vmo);
106 if (Status != ZX_OK) {
108 Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", S,
109 _zx_status_get_string(Status));
113 uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
114 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
116 if (UNLIKELY(internal_iserror(B, &ErrNo))) {
119 "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
124 return reinterpret_cast<T *>(B);
127 template <class T> void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT {
130 uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
131 #if SANITIZER_FUCHSIA
132 _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B),
135 internal_munmap(B, RoundedSize);
139 template <class T, class... U>
140 T *initArray(size_t N, U &&... Us) XRAY_NEVER_INSTRUMENT {
141 auto A = allocateBuffer<T>(N);
144 new (A + (--N)) T(std::forward<U>(Us)...);
148 /// The Allocator type hands out fixed-sized chunks of memory that are
149 /// cache-line aligned and sized. This is useful for placement of
150 /// performance-sensitive data in memory that's frequently accessed. The
151 /// allocator also self-limits the peak memory usage to a dynamically defined
154 /// N is the lower-bound size of the block of memory to return from the
155 /// allocation function. N is used to compute the size of a block, which is
156 /// cache-line-size multiples worth of memory. We compute the size of a block by
157 /// determining how many cache lines worth of memory is required to subsume N.
159 /// The Allocator instance will manage its own memory acquired through mmap.
160 /// This severely constrains the platforms on which this can be used to POSIX
161 /// systems where mmap semantics are well-defined.
163 /// FIXME: Isolate the lower-level memory management to a different abstraction
164 /// that can be platform-specific.
165 template <size_t N> struct Allocator {
166 // The Allocator returns memory as Block instances.
168 /// Compute the minimum cache-line size multiple that is >= N.
169 static constexpr auto Size = nearest_boundary(N, kCacheLineSize);
175 unsigned char *BackingStore = nullptr;
176 unsigned char *AlignedNextBlock = nullptr;
177 size_t AllocatedBlocks = 0;
181 void *Alloc() XRAY_NEVER_INSTRUMENT {
182 SpinMutexLock Lock(&Mutex);
183 if (UNLIKELY(BackingStore == nullptr)) {
184 BackingStore = allocateBuffer(MaxMemory);
185 if (BackingStore == nullptr) {
187 Report("XRay Profiling: Failed to allocate memory for allocator.\n");
191 AlignedNextBlock = BackingStore;
193 // Ensure that NextBlock is aligned appropriately.
194 auto BackingStoreNum = reinterpret_cast<uintptr_t>(BackingStore);
195 auto AlignedNextBlockNum = nearest_boundary(
196 reinterpret_cast<uintptr_t>(AlignedNextBlock), kCacheLineSize);
197 if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) {
198 deallocateBuffer(BackingStore, MaxMemory);
199 AlignedNextBlock = BackingStore = nullptr;
201 Report("XRay Profiling: Cannot obtain enough memory from "
202 "preallocated region.\n");
206 AlignedNextBlock = reinterpret_cast<unsigned char *>(AlignedNextBlockNum);
208 // Assert that AlignedNextBlock is cache-line aligned.
209 DCHECK_EQ(reinterpret_cast<uintptr_t>(AlignedNextBlock) % kCacheLineSize,
213 if (((AllocatedBlocks + 1) * Block::Size) > MaxMemory)
216 // Align the pointer we'd like to return to an appropriate alignment, then
217 // advance the pointer from where to start allocations.
218 void *Result = AlignedNextBlock;
220 reinterpret_cast<unsigned char *>(AlignedNextBlock) + Block::Size;
226 explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT
227 : MaxMemory(RoundUpTo(M, kCacheLineSize)),
228 BackingStore(nullptr),
229 AlignedNextBlock(nullptr),
234 explicit Allocator(void *P, size_t M) XRAY_NEVER_INSTRUMENT
236 BackingStore(reinterpret_cast<unsigned char *>(P)),
237 AlignedNextBlock(reinterpret_cast<unsigned char *>(P)),
242 Allocator(const Allocator &) = delete;
243 Allocator &operator=(const Allocator &) = delete;
245 Allocator(Allocator &&O) XRAY_NEVER_INSTRUMENT {
246 SpinMutexLock L0(&Mutex);
247 SpinMutexLock L1(&O.Mutex);
248 MaxMemory = O.MaxMemory;
250 BackingStore = O.BackingStore;
251 O.BackingStore = nullptr;
252 AlignedNextBlock = O.AlignedNextBlock;
253 O.AlignedNextBlock = nullptr;
254 AllocatedBlocks = O.AllocatedBlocks;
255 O.AllocatedBlocks = 0;
260 Allocator &operator=(Allocator &&O) XRAY_NEVER_INSTRUMENT {
261 SpinMutexLock L0(&Mutex);
262 SpinMutexLock L1(&O.Mutex);
263 MaxMemory = O.MaxMemory;
265 if (BackingStore != nullptr)
266 deallocateBuffer(BackingStore, MaxMemory);
267 BackingStore = O.BackingStore;
268 O.BackingStore = nullptr;
269 AlignedNextBlock = O.AlignedNextBlock;
270 O.AlignedNextBlock = nullptr;
271 AllocatedBlocks = O.AllocatedBlocks;
272 O.AllocatedBlocks = 0;
278 Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; }
280 ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT {
281 if (Owned && BackingStore != nullptr) {
282 deallocateBuffer(BackingStore, MaxMemory);
287 } // namespace __xray
289 #endif // XRAY_ALLOCATOR_H