1 //===-- xray_buffer_queue.h ------------------------------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of XRay, a dynamic runtime instrumentation system.
12 // Defines the interface for a buffer queue implementation.
14 //===----------------------------------------------------------------------===//
15 #ifndef XRAY_BUFFER_QUEUE_H
16 #define XRAY_BUFFER_QUEUE_H
18 #include "sanitizer_common/sanitizer_atomic.h"
19 #include "sanitizer_common/sanitizer_common.h"
20 #include "sanitizer_common/sanitizer_mutex.h"
21 #include "xray_defs.h"
27 /// BufferQueue implements a circular queue of fixed sized buffers (much like a
28 /// freelist) but is concerned with making it quick to initialise, finalise, and
29 /// get from or return buffers to the queue. This is one key component of the
30 /// "flight data recorder" (FDR) mode to support ongoing XRay function call
34 /// ControlBlock represents the memory layout of how we interpret the backing
35 /// store for all buffers and extents managed by a BufferQueue instance. The
36 /// ControlBlock has the reference count as the first member, sized according
37 /// to platform-specific cache-line size. We never use the Buffer member of
38 /// the union, which is only there for compiler-supported alignment and
41 /// This ensures that the `Data` member will be placed at least kCacheLineSize
42 /// bytes from the beginning of the structure.
45 atomic_uint64_t RefCount;
46 char Buffer[kCacheLineSize];
49 /// We need to make this size 1, to conform to the C++ rules for array data
50 /// members. Typically, we want to subtract this 1 byte for sizing
56 atomic_uint64_t *Extents = nullptr;
57 uint64_t Generation{0};
62 friend class BufferQueue;
63 ControlBlock *BackingStore = nullptr;
64 ControlBlock *ExtentsBackingStore = nullptr;
69 // The managed buffer.
72 // This is true if the buffer has been returned to the available queue, and
73 // is considered "used" by another thread.
78 // This models a ForwardIterator. |T| Must be either a `Buffer` or `const
79 // Buffer`. Note that we only advance to the "used" buffers, when
80 // incrementing, so that at dereference we're always at a valid point.
81 template <class T> class Iterator {
83 BufferRep *Buffers = nullptr;
87 Iterator &operator++() {
88 DCHECK_NE(Offset, Max);
91 } while (!Buffers[Offset].Used && Offset != Max);
95 Iterator operator++(int) {
101 T &operator*() const { return Buffers[Offset].Buff; }
103 T *operator->() const { return &(Buffers[Offset].Buff); }
105 Iterator(BufferRep *Root, size_t O, size_t M) XRAY_NEVER_INSTRUMENT
109 // We want to advance to the first Offset where the 'Used' property is
110 // true, or to the end of the list/queue.
111 while (!Buffers[Offset].Used && Offset != Max) {
116 Iterator() = default;
117 Iterator(const Iterator &) = default;
118 Iterator(Iterator &&) = default;
119 Iterator &operator=(const Iterator &) = default;
120 Iterator &operator=(Iterator &&) = default;
121 ~Iterator() = default;
124 friend bool operator==(const Iterator &L, const Iterator<V> &R) {
125 DCHECK_EQ(L.Max, R.Max);
126 return L.Buffers == R.Buffers && L.Offset == R.Offset;
130 friend bool operator!=(const Iterator &L, const Iterator<V> &R) {
135 // Size of each individual Buffer.
138 // Amount of pre-allocated buffers.
142 atomic_uint8_t Finalizing;
144 // The collocated ControlBlock and buffer storage.
145 ControlBlock *BackingStore;
147 // The collocated ControlBlock and extents storage.
148 ControlBlock *ExtentsBackingStore;
150 // A dynamically allocated array of BufferRep instances.
153 // Pointer to the next buffer to be handed out.
156 // Pointer to the entry in the array where the next released buffer will be
160 // Count of buffers that have been handed out through 'getBuffer'.
163 // We use a generation number to identify buffers and which generation they're
165 atomic_uint64_t Generation;
167 /// Releases references to the buffers backed by the current buffer queue.
168 void cleanupBuffers();
171 enum class ErrorCode : unsigned {
180 static const char *getErrorString(ErrorCode E) {
184 case ErrorCode::NotEnoughMemory:
185 return "no available buffers in the queue";
186 case ErrorCode::QueueFinalizing:
187 return "queue already finalizing";
188 case ErrorCode::UnrecognizedBuffer:
189 return "buffer being returned not owned by buffer queue";
190 case ErrorCode::AlreadyFinalized:
191 return "queue already finalized";
192 case ErrorCode::AlreadyInitialized:
193 return "queue already initialized";
195 return "unknown error";
198 /// Initialise a queue of size |N| with buffers of size |B|. We report success
199 /// through |Success|.
200 BufferQueue(size_t B, size_t N, bool &Success);
202 /// Updates |Buf| to contain the pointer to an appropriate buffer. Returns an
203 /// error in case there are no available buffers to return when we will run
204 /// over the upper bound for the total buffers.
207 /// - BufferQueue is not finalising.
210 /// - ErrorCode::NotEnoughMemory on exceeding MaxSize.
211 /// - ErrorCode::Ok when we find a Buffer.
212 /// - ErrorCode::QueueFinalizing or ErrorCode::AlreadyFinalized on
213 /// a finalizing/finalized BufferQueue.
214 ErrorCode getBuffer(Buffer &Buf);
216 /// Updates |Buf| to point to nullptr, with size 0.
219 /// - ErrorCode::Ok when we successfully release the buffer.
220 /// - ErrorCode::UnrecognizedBuffer for when this BufferQueue does not own
221 /// the buffer being released.
222 ErrorCode releaseBuffer(Buffer &Buf);
224 /// Initializes the buffer queue, starting a new generation. We can re-set the
225 /// size of buffers with |BS| along with the buffer count with |BC|.
228 /// - ErrorCode::Ok when we successfully initialize the buffer. This
229 /// requires that the buffer queue is previously finalized.
230 /// - ErrorCode::AlreadyInitialized when the buffer queue is not finalized.
231 ErrorCode init(size_t BS, size_t BC);
233 bool finalizing() const {
234 return atomic_load(&Finalizing, memory_order_acquire);
237 uint64_t generation() const {
238 return atomic_load(&Generation, memory_order_acquire);
241 /// Returns the configured size of the buffers in the buffer queue.
242 size_t ConfiguredBufferSize() const { return BufferSize; }
244 /// Sets the state of the BufferQueue to finalizing, which ensures that:
246 /// - All subsequent attempts to retrieve a Buffer will fail.
247 /// - All releaseBuffer operations will not fail.
249 /// After a call to finalize succeeds, all subsequent calls to finalize will
250 /// fail with ErrorCode::QueueFinalizing.
251 ErrorCode finalize();
253 /// Applies the provided function F to each Buffer in the queue, only if the
254 /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a
255 /// releaseBuffer(...) operation).
256 template <class F> void apply(F Fn) XRAY_NEVER_INSTRUMENT {
257 SpinMutexLock G(&Mutex);
258 for (auto I = begin(), E = end(); I != E; ++I)
262 using const_iterator = Iterator<const Buffer>;
263 using iterator = Iterator<Buffer>;
265 /// Provides iterator access to the raw Buffer instances.
266 iterator begin() const { return iterator(Buffers, 0, BufferCount); }
267 const_iterator cbegin() const {
268 return const_iterator(Buffers, 0, BufferCount);
270 iterator end() const { return iterator(Buffers, BufferCount, BufferCount); }
271 const_iterator cend() const {
272 return const_iterator(Buffers, BufferCount, BufferCount);
275 // Cleans up allocated buffers.
279 } // namespace __xray
281 #endif // XRAY_BUFFER_QUEUE_H