1 //===-- xray_fdr_controller.h ---------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of XRay, a function call tracing system.
12 //===----------------------------------------------------------------------===//
13 #ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_
14 #define COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_
19 #include "xray/xray_interface.h"
20 #include "xray/xray_records.h"
21 #include "xray_buffer_queue.h"
22 #include "xray_fdr_log_writer.h"
26 template <size_t Version = 5> class FDRController {
28 BufferQueue::Buffer &B;
30 int (*WallClockReader)(clockid_t, struct timespec *) = 0;
31 uint64_t CycleThreshold = 0;
33 uint64_t LastFunctionEntryTSC = 0;
34 uint64_t LatestTSC = 0;
35 uint16_t LatestCPU = 0;
40 uint32_t UndoableFunctionEnters = 0;
41 uint32_t UndoableTailExits = 0;
43 bool finalized() const XRAY_NEVER_INSTRUMENT {
44 return BQ == nullptr || BQ->finalizing();
47 bool hasSpace(size_t S) XRAY_NEVER_INSTRUMENT {
48 return B.Data != nullptr && B.Generation == BQ->generation() &&
49 W.getNextRecord() + S <= reinterpret_cast<char *>(B.Data) + B.Size;
52 constexpr int32_t mask(int32_t FuncId) const XRAY_NEVER_INSTRUMENT {
53 return FuncId & ((1 << 29) - 1);
56 bool getNewBuffer() XRAY_NEVER_INSTRUMENT {
57 if (BQ->getBuffer(B) != BufferQueue::ErrorCode::Ok)
61 DCHECK_EQ(W.getNextRecord(), B.Data);
65 UndoableFunctionEnters = 0;
66 UndoableTailExits = 0;
67 atomic_store(B.Extents, 0, memory_order_release);
71 bool setupNewBuffer() XRAY_NEVER_INSTRUMENT {
75 DCHECK(hasSpace(sizeof(MetadataRecord) * 3));
77 PId = internal_getpid();
81 WallClockReader(CLOCK_MONOTONIC, &TS);
83 MetadataRecord Metadata[] = {
84 // Write out a MetadataRecord to signify that this is the start of a new
85 // buffer, associated with a particular thread, with a new CPU. For the
86 // data, we have 15 bytes to squeeze as much information as we can. At
87 // this point we only write down the following bytes:
88 // - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8
90 createMetadataRecord<MetadataRecord::RecordKinds::NewBuffer>(
91 static_cast<int32_t>(TId)),
93 // Also write the WalltimeMarker record. We only really need microsecond
94 // precision here, and enforce across platforms that we need 64-bit
95 // seconds and 32-bit microseconds encoded in the Metadata record.
96 createMetadataRecord<MetadataRecord::RecordKinds::WalltimeMarker>(
97 static_cast<int64_t>(TS.tv_sec),
98 static_cast<int32_t>(TS.tv_nsec / 1000)),
100 // Also write the Pid record.
101 createMetadataRecord<MetadataRecord::RecordKinds::Pid>(
102 static_cast<int32_t>(PId)),
107 return W.writeMetadataRecords(Metadata);
110 bool prepareBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
112 return returnBuffer();
114 if (UNLIKELY(!hasSpace(S))) {
119 if (!setupNewBuffer())
126 atomic_store(B.Extents, 0, memory_order_release);
127 return setupNewBuffer();
133 bool returnBuffer() XRAY_NEVER_INSTRUMENT {
139 BQ->releaseBuffer(B); // ignore result.
143 return BQ->releaseBuffer(B) == BufferQueue::ErrorCode::Ok;
146 enum class PreambleResult { NoChange, WroteMetadata, InvalidBuffer };
147 PreambleResult recordPreamble(uint64_t TSC,
148 uint16_t CPU) XRAY_NEVER_INSTRUMENT {
149 if (UNLIKELY(LatestCPU != CPU || LatestTSC == 0)) {
150 // We update our internal tracking state for the Latest TSC and CPU we've
151 // seen, then write out the appropriate metadata and function records.
155 if (B.Generation != BQ->generation())
156 return PreambleResult::InvalidBuffer;
158 W.writeMetadata<MetadataRecord::RecordKinds::NewCPUId>(CPU, TSC);
159 return PreambleResult::WroteMetadata;
162 DCHECK_EQ(LatestCPU, CPU);
164 if (UNLIKELY(LatestTSC > TSC ||
166 uint64_t{std::numeric_limits<int32_t>::max()})) {
167 // Either the TSC has wrapped around from the last TSC we've seen or the
168 // delta is too large to fit in a 32-bit signed integer, so we write a
169 // wrap-around record.
172 if (B.Generation != BQ->generation())
173 return PreambleResult::InvalidBuffer;
175 W.writeMetadata<MetadataRecord::RecordKinds::TSCWrap>(TSC);
176 return PreambleResult::WroteMetadata;
179 return PreambleResult::NoChange;
182 bool rewindRecords(int32_t FuncId, uint64_t TSC,
183 uint16_t CPU) XRAY_NEVER_INSTRUMENT {
184 // Undo one enter record, because at this point we are either at the state
186 // - We are exiting a function that we recently entered.
187 // - We are exiting a function that was the result of a sequence of tail
188 // exits, and we can check whether the tail exits can be re-wound.
191 W.undoWrites(sizeof(FunctionRecord));
192 if (B.Generation != BQ->generation())
194 internal_memcpy(&F, W.getNextRecord(), sizeof(FunctionRecord));
196 DCHECK(F.RecordKind ==
197 uint8_t(FunctionRecord::RecordKinds::FunctionEnter) &&
198 "Expected to find function entry recording when rewinding.");
199 DCHECK_EQ(F.FuncId, FuncId & ~(0x0F << 28));
201 LatestTSC -= F.TSCDelta;
202 if (--UndoableFunctionEnters != 0) {
203 LastFunctionEntryTSC -= F.TSCDelta;
207 LastFunctionEntryTSC = 0;
208 auto RewindingTSC = LatestTSC;
209 auto RewindingRecordPtr = W.getNextRecord() - sizeof(FunctionRecord);
210 while (UndoableTailExits) {
211 if (B.Generation != BQ->generation())
213 internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord));
214 DCHECK_EQ(F.RecordKind,
215 uint8_t(FunctionRecord::RecordKinds::FunctionTailExit));
216 RewindingTSC -= F.TSCDelta;
217 RewindingRecordPtr -= sizeof(FunctionRecord);
218 if (B.Generation != BQ->generation())
220 internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord));
222 // This tail call exceeded the threshold duration. It will not be erased.
223 if ((TSC - RewindingTSC) >= CycleThreshold) {
224 UndoableTailExits = 0;
229 W.undoWrites(sizeof(FunctionRecord) * 2);
230 LatestTSC = RewindingTSC;
236 template <class WallClockFunc>
237 FDRController(BufferQueue *BQ, BufferQueue::Buffer &B, FDRLogWriter &W,
238 WallClockFunc R, uint64_t C) XRAY_NEVER_INSTRUMENT
245 bool functionEnter(int32_t FuncId, uint64_t TSC,
246 uint16_t CPU) XRAY_NEVER_INSTRUMENT {
248 !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord)))
249 return returnBuffer();
251 auto PreambleStatus = recordPreamble(TSC, CPU);
252 if (PreambleStatus == PreambleResult::InvalidBuffer)
253 return returnBuffer();
255 if (PreambleStatus == PreambleResult::WroteMetadata) {
256 UndoableFunctionEnters = 1;
257 UndoableTailExits = 0;
259 ++UndoableFunctionEnters;
262 auto Delta = TSC - LatestTSC;
263 LastFunctionEntryTSC = TSC;
265 return W.writeFunction(FDRLogWriter::FunctionRecordKind::Enter,
266 mask(FuncId), Delta);
269 bool functionTailExit(int32_t FuncId, uint64_t TSC,
270 uint16_t CPU) XRAY_NEVER_INSTRUMENT {
272 return returnBuffer();
274 if (!prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord)))
275 return returnBuffer();
277 auto PreambleStatus = recordPreamble(TSC, CPU);
278 if (PreambleStatus == PreambleResult::InvalidBuffer)
279 return returnBuffer();
281 if (PreambleStatus == PreambleResult::NoChange &&
282 UndoableFunctionEnters != 0 &&
283 TSC - LastFunctionEntryTSC < CycleThreshold)
284 return rewindRecords(FuncId, TSC, CPU);
286 UndoableTailExits = UndoableFunctionEnters ? UndoableTailExits + 1 : 0;
287 UndoableFunctionEnters = 0;
288 auto Delta = TSC - LatestTSC;
290 return W.writeFunction(FDRLogWriter::FunctionRecordKind::TailExit,
291 mask(FuncId), Delta);
294 bool functionEnterArg(int32_t FuncId, uint64_t TSC, uint16_t CPU,
295 uint64_t Arg) XRAY_NEVER_INSTRUMENT {
297 !prepareBuffer((2 * sizeof(MetadataRecord)) + sizeof(FunctionRecord)) ||
298 recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer)
299 return returnBuffer();
301 auto Delta = TSC - LatestTSC;
303 LastFunctionEntryTSC = 0;
304 UndoableFunctionEnters = 0;
305 UndoableTailExits = 0;
307 return W.writeFunctionWithArg(FDRLogWriter::FunctionRecordKind::EnterArg,
308 mask(FuncId), Delta, Arg);
311 bool functionExit(int32_t FuncId, uint64_t TSC,
312 uint16_t CPU) XRAY_NEVER_INSTRUMENT {
314 !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord)))
315 return returnBuffer();
317 auto PreambleStatus = recordPreamble(TSC, CPU);
318 if (PreambleStatus == PreambleResult::InvalidBuffer)
319 return returnBuffer();
321 if (PreambleStatus == PreambleResult::NoChange &&
322 UndoableFunctionEnters != 0 &&
323 TSC - LastFunctionEntryTSC < CycleThreshold)
324 return rewindRecords(FuncId, TSC, CPU);
326 auto Delta = TSC - LatestTSC;
328 UndoableFunctionEnters = 0;
329 UndoableTailExits = 0;
330 return W.writeFunction(FDRLogWriter::FunctionRecordKind::Exit, mask(FuncId),
334 bool customEvent(uint64_t TSC, uint16_t CPU, const void *Event,
335 int32_t EventSize) XRAY_NEVER_INSTRUMENT {
337 !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) ||
338 recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer)
339 return returnBuffer();
341 auto Delta = TSC - LatestTSC;
343 UndoableFunctionEnters = 0;
344 UndoableTailExits = 0;
345 return W.writeCustomEvent(Delta, Event, EventSize);
348 bool typedEvent(uint64_t TSC, uint16_t CPU, uint16_t EventType,
349 const void *Event, int32_t EventSize) XRAY_NEVER_INSTRUMENT {
351 !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) ||
352 recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer)
353 return returnBuffer();
355 auto Delta = TSC - LatestTSC;
357 UndoableFunctionEnters = 0;
358 UndoableTailExits = 0;
359 return W.writeTypedEvent(Delta, EventType, Event, EventSize);
362 bool flush() XRAY_NEVER_INSTRUMENT {
364 returnBuffer(); // ignore result.
367 return returnBuffer();
371 } // namespace __xray
373 #endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_