1 //===- xray-converter.cpp: XRay Trace Conversion --------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Implements the trace conversion functions.
12 //===----------------------------------------------------------------------===//
13 #include "xray-converter.h"
15 #include "trie-node.h"
16 #include "xray-registry.h"
17 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
18 #include "llvm/Support/EndianStream.h"
19 #include "llvm/Support/FileSystem.h"
20 #include "llvm/Support/FormatVariadic.h"
21 #include "llvm/Support/ScopedPrinter.h"
22 #include "llvm/Support/YAMLTraits.h"
23 #include "llvm/Support/raw_ostream.h"
24 #include "llvm/XRay/InstrumentationMap.h"
25 #include "llvm/XRay/Trace.h"
26 #include "llvm/XRay/YAMLXRayRecord.h"
32 // ----------------------------------------------------------------------------
33 static cl::SubCommand Convert("convert", "Trace Format Conversion");
34 static cl::opt<std::string> ConvertInput(cl::Positional,
35 cl::desc("<xray log file>"),
36 cl::Required, cl::sub(Convert));
37 enum class ConvertFormats { BINARY, YAML, CHROME_TRACE_EVENT };
38 static cl::opt<ConvertFormats> ConvertOutputFormat(
39 "output-format", cl::desc("output format"),
40 cl::values(clEnumValN(ConvertFormats::BINARY, "raw", "output in binary"),
41 clEnumValN(ConvertFormats::YAML, "yaml", "output in yaml"),
42 clEnumValN(ConvertFormats::CHROME_TRACE_EVENT, "trace_event",
43 "Output in chrome's trace event format. "
44 "May be visualized with the Catapult trace viewer.")),
46 static cl::alias ConvertOutputFormat2("f", cl::aliasopt(ConvertOutputFormat),
47 cl::desc("Alias for -output-format"),
49 static cl::opt<std::string>
50 ConvertOutput("output", cl::value_desc("output file"), cl::init("-"),
51 cl::desc("output file; use '-' for stdout"),
53 static cl::alias ConvertOutput2("o", cl::aliasopt(ConvertOutput),
54 cl::desc("Alias for -output"),
58 ConvertSymbolize("symbolize",
59 cl::desc("symbolize function ids from the input log"),
60 cl::init(false), cl::sub(Convert));
61 static cl::alias ConvertSymbolize2("y", cl::aliasopt(ConvertSymbolize),
62 cl::desc("Alias for -symbolize"),
65 static cl::opt<std::string>
66 ConvertInstrMap("instr_map",
67 cl::desc("binary with the instrumentation map, or "
68 "a separate instrumentation map"),
69 cl::value_desc("binary with xray_instr_map"),
70 cl::sub(Convert), cl::init(""));
71 static cl::alias ConvertInstrMap2("m", cl::aliasopt(ConvertInstrMap),
72 cl::desc("Alias for -instr_map"),
74 static cl::opt<bool> ConvertSortInput(
76 cl::desc("determines whether to sort input log records by timestamp"),
77 cl::sub(Convert), cl::init(true));
78 static cl::alias ConvertSortInput2("s", cl::aliasopt(ConvertSortInput),
79 cl::desc("Alias for -sort"),
82 using llvm::yaml::Output;
84 void TraceConverter::exportAsYAML(const Trace &Records, raw_ostream &OS) {
86 const auto &FH = Records.getFileHeader();
87 Trace.Header = {FH.Version, FH.Type, FH.ConstantTSC, FH.NonstopTSC,
89 Trace.Records.reserve(Records.size());
90 for (const auto &R : Records) {
91 Trace.Records.push_back({R.RecordType, R.CPU, R.Type, R.FuncId,
92 Symbolize ? FuncIdHelper.SymbolOrNumber(R.FuncId)
93 : llvm::to_string(R.FuncId),
94 R.TSC, R.TId, R.PId, R.CallArgs});
96 Output Out(OS, nullptr, 0);
100 void TraceConverter::exportAsRAWv1(const Trace &Records, raw_ostream &OS) {
101 // First write out the file header, in the correct endian-appropriate format
102 // (XRay assumes currently little endian).
103 support::endian::Writer Writer(OS, support::endianness::little);
104 const auto &FH = Records.getFileHeader();
105 Writer.write(FH.Version);
106 Writer.write(FH.Type);
107 uint32_t Bitfield{0};
111 Bitfield |= 1uL << 1;
112 Writer.write(Bitfield);
113 Writer.write(FH.CycleFrequency);
115 // There's 16 bytes of padding at the end of the file header.
116 static constexpr uint32_t Padding4B = 0;
117 Writer.write(Padding4B);
118 Writer.write(Padding4B);
119 Writer.write(Padding4B);
120 Writer.write(Padding4B);
122 // Then write out the rest of the records, still in an endian-appropriate
124 for (const auto &R : Records) {
125 Writer.write(R.RecordType);
126 // The on disk naive raw format uses 8 bit CPUs, but the record has 16.
127 // There's no choice but truncation.
128 Writer.write(static_cast<uint8_t>(R.CPU));
130 case RecordTypes::ENTER:
131 case RecordTypes::ENTER_ARG:
132 Writer.write(uint8_t{0});
134 case RecordTypes::EXIT:
135 Writer.write(uint8_t{1});
137 case RecordTypes::TAIL_EXIT:
138 Writer.write(uint8_t{2});
141 Writer.write(R.FuncId);
148 Writer.write(Padding4B);
150 Writer.write(Padding4B);
151 Writer.write(Padding4B);
157 // A structure that allows building a dictionary of stack ids for the Chrome
158 // trace event format.
160 // Each Stack of function calls has a unique ID.
163 // Bookkeeping so that IDs can be maintained uniquely across threads.
164 // Traversal keeps sibling pointers to other threads stacks. This is helpful
165 // to determine when a thread encounters a new stack and should assign a new
167 SmallVector<TrieNode<StackIdData> *, 4> siblings;
170 using StackTrieNode = TrieNode<StackIdData>;
172 // A helper function to find the sibling nodes for an encountered function in a
173 // thread of execution. Relies on the invariant that each time a new node is
174 // traversed in a thread, sibling bidirectional pointers are maintained.
175 SmallVector<StackTrieNode *, 4>
176 findSiblings(StackTrieNode *parent, int32_t FnId, uint32_t TId,
177 const DenseMap<uint32_t, SmallVector<StackTrieNode *, 4>>
178 &StackRootsByThreadId) {
180 SmallVector<StackTrieNode *, 4> Siblings{};
182 if (parent == nullptr) {
183 for (auto map_iter : StackRootsByThreadId) {
184 // Only look for siblings in other threads.
185 if (map_iter.first != TId)
186 for (auto node_iter : map_iter.second) {
187 if (node_iter->FuncId == FnId)
188 Siblings.push_back(node_iter);
194 for (auto *ParentSibling : parent->ExtraData.siblings)
195 for (auto node_iter : ParentSibling->Callees)
196 if (node_iter->FuncId == FnId)
197 Siblings.push_back(node_iter);
202 // Given a function being invoked in a thread with id TId, finds and returns the
203 // StackTrie representing the function call stack. If no node exists, creates
204 // the node. Assigns unique IDs to stacks newly encountered among all threads
205 // and keeps sibling links up to when creating new nodes.
206 StackTrieNode *findOrCreateStackNode(
207 StackTrieNode *Parent, int32_t FuncId, uint32_t TId,
208 DenseMap<uint32_t, SmallVector<StackTrieNode *, 4>> &StackRootsByThreadId,
209 DenseMap<unsigned, StackTrieNode *> &StacksByStackId, unsigned *id_counter,
210 std::forward_list<StackTrieNode> &NodeStore) {
211 SmallVector<StackTrieNode *, 4> &ParentCallees =
212 Parent == nullptr ? StackRootsByThreadId[TId] : Parent->Callees;
213 auto match = find_if(ParentCallees, [FuncId](StackTrieNode *ParentCallee) {
214 return FuncId == ParentCallee->FuncId;
216 if (match != ParentCallees.end())
219 SmallVector<StackTrieNode *, 4> siblings =
220 findSiblings(Parent, FuncId, TId, StackRootsByThreadId);
221 if (siblings.empty()) {
222 NodeStore.push_front({FuncId, Parent, {}, {(*id_counter)++, {}}});
223 StackTrieNode *CurrentStack = &NodeStore.front();
224 StacksByStackId[*id_counter - 1] = CurrentStack;
225 ParentCallees.push_back(CurrentStack);
228 unsigned stack_id = siblings[0]->ExtraData.id;
229 NodeStore.push_front({FuncId, Parent, {}, {stack_id, std::move(siblings)}});
230 StackTrieNode *CurrentStack = &NodeStore.front();
231 for (auto *sibling : CurrentStack->ExtraData.siblings)
232 sibling->ExtraData.siblings.push_back(CurrentStack);
233 ParentCallees.push_back(CurrentStack);
237 void writeTraceViewerRecord(uint16_t Version, raw_ostream &OS, int32_t FuncId,
238 uint32_t TId, uint32_t PId, bool Symbolize,
239 const FuncIdConversionHelper &FuncIdHelper,
240 double EventTimestampUs,
241 const StackTrieNode &StackCursor,
242 StringRef FunctionPhenotype) {
246 R"({ "name" : "{0}", "ph" : "{1}", "tid" : "{2}", "pid" : "{3}", )"
247 R"("ts" : "{4:f4}", "sf" : "{5}" })",
248 (Symbolize ? FuncIdHelper.SymbolOrNumber(FuncId)
249 : llvm::to_string(FuncId)),
250 FunctionPhenotype, TId, PId, EventTimestampUs,
251 StackCursor.ExtraData.id);
254 R"({ "name" : "{0}", "ph" : "{1}", "tid" : "{2}", "pid" : "1", )"
255 R"("ts" : "{3:f3}", "sf" : "{4}" })",
256 (Symbolize ? FuncIdHelper.SymbolOrNumber(FuncId)
257 : llvm::to_string(FuncId)),
258 FunctionPhenotype, TId, EventTimestampUs, StackCursor.ExtraData.id);
264 void TraceConverter::exportAsChromeTraceEventFormat(const Trace &Records,
266 const auto &FH = Records.getFileHeader();
267 auto Version = FH.Version;
268 auto CycleFreq = FH.CycleFrequency;
270 unsigned id_counter = 0;
272 OS << "{\n \"traceEvents\": [";
273 DenseMap<uint32_t, StackTrieNode *> StackCursorByThreadId{};
274 DenseMap<uint32_t, SmallVector<StackTrieNode *, 4>> StackRootsByThreadId{};
275 DenseMap<unsigned, StackTrieNode *> StacksByStackId{};
276 std::forward_list<StackTrieNode> NodeStore{};
278 for (const auto &R : Records) {
279 if (loop_count++ == 0)
284 // Chrome trace event format always wants data in micros.
285 // CyclesPerMicro = CycleHertz / 10^6
286 // TSC / CyclesPerMicro == TSC * 10^6 / CycleHertz == MicroTimestamp
287 // Could lose some precision here by converting the TSC to a double to
288 // multiply by the period in micros. 52 bit mantissa is a good start though.
289 // TODO: Make feature request to Chrome Trace viewer to accept ticks and a
290 // frequency or do some more involved calculation to avoid dangers of
292 double EventTimestampUs = double(1000000) / CycleFreq * double(R.TSC);
293 StackTrieNode *&StackCursor = StackCursorByThreadId[R.TId];
295 case RecordTypes::ENTER:
296 case RecordTypes::ENTER_ARG:
297 StackCursor = findOrCreateStackNode(StackCursor, R.FuncId, R.TId,
298 StackRootsByThreadId, StacksByStackId,
299 &id_counter, NodeStore);
300 // Each record is represented as a json dictionary with function name,
301 // type of B for begin or E for end, thread id, process id,
302 // timestamp in microseconds, and a stack frame id. The ids are logged
303 // in an id dictionary after the events.
304 writeTraceViewerRecord(Version, OS, R.FuncId, R.TId, R.PId, Symbolize,
305 FuncIdHelper, EventTimestampUs, *StackCursor, "B");
307 case RecordTypes::EXIT:
308 case RecordTypes::TAIL_EXIT:
309 // No entries to record end for.
310 if (StackCursor == nullptr)
312 // Should we emit an END record anyway or account this condition?
313 // (And/Or in loop termination below)
314 StackTrieNode *PreviousCursor = nullptr;
316 if (PreviousCursor != nullptr) {
319 writeTraceViewerRecord(Version, OS, StackCursor->FuncId, R.TId, R.PId,
320 Symbolize, FuncIdHelper, EventTimestampUs,
322 PreviousCursor = StackCursor;
323 StackCursor = StackCursor->Parent;
324 } while (PreviousCursor->FuncId != R.FuncId && StackCursor != nullptr);
328 OS << "\n ],\n"; // Close the Trace Events array.
330 << "\"displayTimeUnit\": \"ns\",\n";
332 // The stackFrames dictionary substantially reduces size of the output file by
333 // avoiding repeating the entire call stack of function names for each entry.
334 OS << R"( "stackFrames": {)";
335 int stack_frame_count = 0;
336 for (auto map_iter : StacksByStackId) {
337 if (stack_frame_count++ == 0)
343 R"("{0}" : { "name" : "{1}")", map_iter.first,
344 (Symbolize ? FuncIdHelper.SymbolOrNumber(map_iter.second->FuncId)
345 : llvm::to_string(map_iter.second->FuncId)));
346 if (map_iter.second->Parent != nullptr)
347 OS << llvm::formatv(R"(, "parent": "{0}")",
348 map_iter.second->Parent->ExtraData.id);
351 OS << "\n }\n"; // Close the stack frames map.
352 OS << "}\n"; // Close the JSON entry.
358 static CommandRegistration Unused(&Convert, []() -> Error {
359 // FIXME: Support conversion to BINARY when upgrading XRay trace versions.
360 InstrumentationMap Map;
361 if (!ConvertInstrMap.empty()) {
362 auto InstrumentationMapOrError = loadInstrumentationMap(ConvertInstrMap);
363 if (!InstrumentationMapOrError)
364 return joinErrors(make_error<StringError>(
365 Twine("Cannot open instrumentation map '") +
366 ConvertInstrMap + "'",
367 std::make_error_code(std::errc::invalid_argument)),
368 InstrumentationMapOrError.takeError());
369 Map = std::move(*InstrumentationMapOrError);
372 const auto &FunctionAddresses = Map.getFunctionAddresses();
373 symbolize::LLVMSymbolizer::Options Opts(
374 symbolize::FunctionNameKind::LinkageName, true, true, false, "");
375 symbolize::LLVMSymbolizer Symbolizer(Opts);
376 llvm::xray::FuncIdConversionHelper FuncIdHelper(ConvertInstrMap, Symbolizer,
378 llvm::xray::TraceConverter TC(FuncIdHelper, ConvertSymbolize);
380 raw_fd_ostream OS(ConvertOutput, EC,
381 ConvertOutputFormat == ConvertFormats::BINARY
382 ? sys::fs::OpenFlags::F_None
383 : sys::fs::OpenFlags::F_Text);
385 return make_error<StringError>(
386 Twine("Cannot open file '") + ConvertOutput + "' for writing.", EC);
388 auto TraceOrErr = loadTraceFile(ConvertInput, ConvertSortInput);
391 make_error<StringError>(
392 Twine("Failed loading input file '") + ConvertInput + "'.",
393 std::make_error_code(std::errc::executable_format_error)),
394 TraceOrErr.takeError());
396 auto &T = *TraceOrErr;
397 switch (ConvertOutputFormat) {
398 case ConvertFormats::YAML:
399 TC.exportAsYAML(T, OS);
401 case ConvertFormats::BINARY:
402 TC.exportAsRAWv1(T, OS);
404 case ConvertFormats::CHROME_TRACE_EVENT:
405 TC.exportAsChromeTraceEventFormat(T, OS);
408 return Error::success();