1 //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains support for reading profiling data for instrumentation
10 // based PGO and coverage.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_PROFILEDATA_INSTRPROFREADER_H
15 #define LLVM_PROFILEDATA_INSTRPROFREADER_H
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/IR/ProfileSummary.h"
20 #include "llvm/ProfileData/InstrProf.h"
21 #include "llvm/Support/Endian.h"
22 #include "llvm/Support/Error.h"
23 #include "llvm/Support/LineIterator.h"
24 #include "llvm/Support/MemoryBuffer.h"
25 #include "llvm/Support/OnDiskHashTable.h"
26 #include "llvm/Support/SwapByteOrder.h"
38 class InstrProfReader;
40 /// A file format agnostic iterator over profiling data.
41 class InstrProfIterator : public std::iterator<std::input_iterator_tag,
42 NamedInstrProfRecord> {
43 InstrProfReader *Reader = nullptr;
49 InstrProfIterator() = default;
50 InstrProfIterator(InstrProfReader *Reader) : Reader(Reader) { Increment(); }
52 InstrProfIterator &operator++() { Increment(); return *this; }
53 bool operator==(const InstrProfIterator &RHS) { return Reader == RHS.Reader; }
54 bool operator!=(const InstrProfIterator &RHS) { return Reader != RHS.Reader; }
55 value_type &operator*() { return Record; }
56 value_type *operator->() { return &Record; }
59 /// Base class and interface for reading profiling data of any known instrprof
60 /// format. Provides an iterator over NamedInstrProfRecords.
61 class InstrProfReader {
62 instrprof_error LastError = instrprof_error::success;
65 InstrProfReader() = default;
66 virtual ~InstrProfReader() = default;
68 /// Read the header. Required before reading first record.
69 virtual Error readHeader() = 0;
71 /// Read a single record.
72 virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0;
74 /// Iterator over profile data.
75 InstrProfIterator begin() { return InstrProfIterator(this); }
76 InstrProfIterator end() { return InstrProfIterator(); }
78 virtual bool isIRLevelProfile() const = 0;
80 virtual bool hasCSIRLevelProfile() const = 0;
82 /// Return the PGO symtab. There are three different readers:
83 /// Raw, Text, and Indexed profile readers. The first two types
84 /// of readers are used only by llvm-profdata tool, while the indexed
85 /// profile reader is also used by llvm-cov tool and the compiler (
86 /// backend or frontend). Since creating PGO symtab can create
87 /// significant runtime and memory overhead (as it touches data
88 /// for the whole program), InstrProfSymtab for the indexed profile
89 /// reader should be created on demand and it is recommended to be
90 /// only used for dumping purpose with llvm-proftool, not with the
92 virtual InstrProfSymtab &getSymtab() = 0;
94 /// Compute the sum of counts and return in Sum.
95 void accumuateCounts(CountSumOrPercent &Sum, bool IsCS);
98 std::unique_ptr<InstrProfSymtab> Symtab;
100 /// Set the current error and return same.
101 Error error(instrprof_error Err) {
103 if (Err == instrprof_error::success)
104 return Error::success();
105 return make_error<InstrProfError>(Err);
108 Error error(Error &&E) { return error(InstrProfError::take(std::move(E))); }
110 /// Clear the current error and return a successful one.
111 Error success() { return error(instrprof_error::success); }
114 /// Return true if the reader has finished reading the profile data.
115 bool isEOF() { return LastError == instrprof_error::eof; }
117 /// Return true if the reader encountered an error reading profiling data.
118 bool hasError() { return LastError != instrprof_error::success && !isEOF(); }
120 /// Get the current error.
123 return make_error<InstrProfError>(LastError);
124 return Error::success();
127 /// Factory method to create an appropriately typed reader for the given
129 static Expected<std::unique_ptr<InstrProfReader>> create(const Twine &Path);
131 static Expected<std::unique_ptr<InstrProfReader>>
132 create(std::unique_ptr<MemoryBuffer> Buffer);
135 /// Reader for the simple text based instrprof format.
137 /// This format is a simple text format that's suitable for test data. Records
138 /// are separated by one or more blank lines, and record fields are separated by
141 /// Each record consists of a function name, a function hash, a number of
142 /// counters, and then each counter value, in that order.
143 class TextInstrProfReader : public InstrProfReader {
145 /// The profile data file contents.
146 std::unique_ptr<MemoryBuffer> DataBuffer;
147 /// Iterator over the profile data.
149 bool IsIRLevelProfile = false;
150 bool HasCSIRLevelProfile = false;
152 Error readValueProfileData(InstrProfRecord &Record);
155 TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_)
156 : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {}
157 TextInstrProfReader(const TextInstrProfReader &) = delete;
158 TextInstrProfReader &operator=(const TextInstrProfReader &) = delete;
160 /// Return true if the given buffer is in text instrprof format.
161 static bool hasFormat(const MemoryBuffer &Buffer);
163 bool isIRLevelProfile() const override { return IsIRLevelProfile; }
165 bool hasCSIRLevelProfile() const override { return HasCSIRLevelProfile; }
168 Error readHeader() override;
170 /// Read a single record.
171 Error readNextRecord(NamedInstrProfRecord &Record) override;
173 InstrProfSymtab &getSymtab() override {
174 assert(Symtab.get());
175 return *Symtab.get();
179 /// Reader for the raw instrprof binary format from runtime.
181 /// This format is a raw memory dump of the instrumentation-baed profiling data
182 /// from the runtime. It has no index.
184 /// Templated on the unsigned type whose size matches pointers on the platform
185 /// that wrote the profile.
186 template <class IntPtrT>
187 class RawInstrProfReader : public InstrProfReader {
189 /// The profile data file contents.
190 std::unique_ptr<MemoryBuffer> DataBuffer;
191 bool ShouldSwapBytes;
192 // The value of the version field of the raw profile data header. The lower 56
193 // bits specifies the format version and the most significant 8 bits specify
194 // the variant types of the profile.
196 uint64_t CountersDelta;
198 const RawInstrProf::ProfileData<IntPtrT> *Data;
199 const RawInstrProf::ProfileData<IntPtrT> *DataEnd;
200 const uint64_t *CountersStart;
201 const char *NamesStart;
203 // After value profile is all read, this pointer points to
204 // the header of next profile data (if exists)
205 const uint8_t *ValueDataStart;
206 uint32_t ValueKindLast;
207 uint32_t CurValueDataSize;
210 RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer)
211 : DataBuffer(std::move(DataBuffer)) {}
212 RawInstrProfReader(const RawInstrProfReader &) = delete;
213 RawInstrProfReader &operator=(const RawInstrProfReader &) = delete;
215 static bool hasFormat(const MemoryBuffer &DataBuffer);
216 Error readHeader() override;
217 Error readNextRecord(NamedInstrProfRecord &Record) override;
219 bool isIRLevelProfile() const override {
220 return (Version & VARIANT_MASK_IR_PROF) != 0;
223 bool hasCSIRLevelProfile() const override {
224 return (Version & VARIANT_MASK_CSIR_PROF) != 0;
227 InstrProfSymtab &getSymtab() override {
228 assert(Symtab.get());
229 return *Symtab.get();
233 Error createSymtab(InstrProfSymtab &Symtab);
234 Error readNextHeader(const char *CurrentPos);
235 Error readHeader(const RawInstrProf::Header &Header);
237 template <class IntT> IntT swap(IntT Int) const {
238 return ShouldSwapBytes ? sys::getSwappedBytes(Int) : Int;
241 support::endianness getDataEndianness() const {
242 support::endianness HostEndian = getHostEndianness();
243 if (!ShouldSwapBytes)
245 if (HostEndian == support::little)
248 return support::little;
251 inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) {
252 return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
255 Error readName(NamedInstrProfRecord &Record);
256 Error readFuncHash(NamedInstrProfRecord &Record);
257 Error readRawCounts(InstrProfRecord &Record);
258 Error readValueProfilingData(InstrProfRecord &Record);
259 bool atEnd() const { return Data == DataEnd; }
263 ValueDataStart += CurValueDataSize;
266 const char *getNextHeaderPos() const {
268 return (const char *)ValueDataStart;
271 const uint64_t *getCounter(IntPtrT CounterPtr) const {
272 ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t);
273 return CountersStart + Offset;
276 StringRef getName(uint64_t NameRef) const {
277 return Symtab->getFuncName(swap(NameRef));
281 using RawInstrProfReader32 = RawInstrProfReader<uint32_t>;
282 using RawInstrProfReader64 = RawInstrProfReader<uint64_t>;
284 namespace IndexedInstrProf {
286 enum class HashT : uint32_t;
288 } // end namespace IndexedInstrProf
290 /// Trait for lookups into the on-disk hash table for the binary instrprof
292 class InstrProfLookupTrait {
293 std::vector<NamedInstrProfRecord> DataBuffer;
294 IndexedInstrProf::HashT HashType;
295 unsigned FormatVersion;
296 // Endianness of the input value profile data.
297 // It should be LE by default, but can be changed
298 // for testing purpose.
299 support::endianness ValueProfDataEndianness = support::little;
302 InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
303 : HashType(HashType), FormatVersion(FormatVersion) {}
305 using data_type = ArrayRef<NamedInstrProfRecord>;
307 using internal_key_type = StringRef;
308 using external_key_type = StringRef;
309 using hash_value_type = uint64_t;
310 using offset_type = uint64_t;
312 static bool EqualKey(StringRef A, StringRef B) { return A == B; }
313 static StringRef GetInternalKey(StringRef K) { return K; }
314 static StringRef GetExternalKey(StringRef K) { return K; }
316 hash_value_type ComputeHash(StringRef K);
318 static std::pair<offset_type, offset_type>
319 ReadKeyDataLength(const unsigned char *&D) {
320 using namespace support;
322 offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
323 offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
324 return std::make_pair(KeyLen, DataLen);
327 StringRef ReadKey(const unsigned char *D, offset_type N) {
328 return StringRef((const char *)D, N);
331 bool readValueProfilingData(const unsigned char *&D,
332 const unsigned char *const End);
333 data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
335 // Used for testing purpose only.
336 void setValueProfDataEndianness(support::endianness Endianness) {
337 ValueProfDataEndianness = Endianness;
341 struct InstrProfReaderIndexBase {
342 virtual ~InstrProfReaderIndexBase() = default;
344 // Read all the profile records with the same key pointed to the current
346 virtual Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) = 0;
348 // Read all the profile records with the key equal to FuncName
349 virtual Error getRecords(StringRef FuncName,
350 ArrayRef<NamedInstrProfRecord> &Data) = 0;
351 virtual void advanceToNextKey() = 0;
352 virtual bool atEnd() const = 0;
353 virtual void setValueProfDataEndianness(support::endianness Endianness) = 0;
354 virtual uint64_t getVersion() const = 0;
355 virtual bool isIRLevelProfile() const = 0;
356 virtual bool hasCSIRLevelProfile() const = 0;
357 virtual Error populateSymtab(InstrProfSymtab &) = 0;
360 using OnDiskHashTableImplV3 =
361 OnDiskIterableChainedHashTable<InstrProfLookupTrait>;
363 template <typename HashTableImpl>
364 class InstrProfReaderItaniumRemapper;
366 template <typename HashTableImpl>
367 class InstrProfReaderIndex : public InstrProfReaderIndexBase {
369 std::unique_ptr<HashTableImpl> HashTable;
370 typename HashTableImpl::data_iterator RecordIterator;
371 uint64_t FormatVersion;
373 friend class InstrProfReaderItaniumRemapper<HashTableImpl>;
376 InstrProfReaderIndex(const unsigned char *Buckets,
377 const unsigned char *const Payload,
378 const unsigned char *const Base,
379 IndexedInstrProf::HashT HashType, uint64_t Version);
380 ~InstrProfReaderIndex() override = default;
382 Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) override;
383 Error getRecords(StringRef FuncName,
384 ArrayRef<NamedInstrProfRecord> &Data) override;
385 void advanceToNextKey() override { RecordIterator++; }
387 bool atEnd() const override {
388 return RecordIterator == HashTable->data_end();
391 void setValueProfDataEndianness(support::endianness Endianness) override {
392 HashTable->getInfoObj().setValueProfDataEndianness(Endianness);
395 uint64_t getVersion() const override { return GET_VERSION(FormatVersion); }
397 bool isIRLevelProfile() const override {
398 return (FormatVersion & VARIANT_MASK_IR_PROF) != 0;
401 bool hasCSIRLevelProfile() const override {
402 return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0;
405 Error populateSymtab(InstrProfSymtab &Symtab) override {
406 return Symtab.create(HashTable->keys());
410 /// Name matcher supporting fuzzy matching of symbol names to names in profiles.
411 class InstrProfReaderRemapper {
413 virtual ~InstrProfReaderRemapper() {}
414 virtual Error populateRemappings() { return Error::success(); }
415 virtual Error getRecords(StringRef FuncName,
416 ArrayRef<NamedInstrProfRecord> &Data) = 0;
419 /// Reader for the indexed binary instrprof format.
420 class IndexedInstrProfReader : public InstrProfReader {
422 /// The profile data file contents.
423 std::unique_ptr<MemoryBuffer> DataBuffer;
424 /// The profile remapping file contents.
425 std::unique_ptr<MemoryBuffer> RemappingBuffer;
426 /// The index into the profile data.
427 std::unique_ptr<InstrProfReaderIndexBase> Index;
428 /// The profile remapping file contents.
429 std::unique_ptr<InstrProfReaderRemapper> Remapper;
430 /// Profile summary data.
431 std::unique_ptr<ProfileSummary> Summary;
432 /// Context sensitive profile summary data.
433 std::unique_ptr<ProfileSummary> CS_Summary;
434 // Index to the current record in the record array.
435 unsigned RecordIndex;
437 // Read the profile summary. Return a pointer pointing to one byte past the
438 // end of the summary data if it exists or the input \c Cur.
439 // \c UseCS indicates whether to use the context-sensitive profile summary.
440 const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version,
441 const unsigned char *Cur, bool UseCS);
444 IndexedInstrProfReader(
445 std::unique_ptr<MemoryBuffer> DataBuffer,
446 std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr)
447 : DataBuffer(std::move(DataBuffer)),
448 RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {}
449 IndexedInstrProfReader(const IndexedInstrProfReader &) = delete;
450 IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete;
452 /// Return the profile version.
453 uint64_t getVersion() const { return Index->getVersion(); }
454 bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); }
455 bool hasCSIRLevelProfile() const override {
456 return Index->hasCSIRLevelProfile();
459 /// Return true if the given buffer is in an indexed instrprof format.
460 static bool hasFormat(const MemoryBuffer &DataBuffer);
462 /// Read the file header.
463 Error readHeader() override;
464 /// Read a single record.
465 Error readNextRecord(NamedInstrProfRecord &Record) override;
467 /// Return the NamedInstrProfRecord associated with FuncName and FuncHash
468 Expected<InstrProfRecord> getInstrProfRecord(StringRef FuncName,
471 /// Fill Counts with the profile data for the given function name.
472 Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash,
473 std::vector<uint64_t> &Counts);
475 /// Return the maximum of all known function counts.
476 /// \c UseCS indicates whether to use the context-sensitive count.
477 uint64_t getMaximumFunctionCount(bool UseCS) {
479 assert(CS_Summary && "No context sensitive profile summary");
480 return CS_Summary->getMaxFunctionCount();
482 assert(Summary && "No profile summary");
483 return Summary->getMaxFunctionCount();
487 /// Factory method to create an indexed reader.
488 static Expected<std::unique_ptr<IndexedInstrProfReader>>
489 create(const Twine &Path, const Twine &RemappingPath = "");
491 static Expected<std::unique_ptr<IndexedInstrProfReader>>
492 create(std::unique_ptr<MemoryBuffer> Buffer,
493 std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr);
495 // Used for testing purpose only.
496 void setValueProfDataEndianness(support::endianness Endianness) {
497 Index->setValueProfDataEndianness(Endianness);
500 // See description in the base class. This interface is designed
501 // to be used by llvm-profdata (for dumping). Avoid using this when
502 // the client is the compiler.
503 InstrProfSymtab &getSymtab() override;
505 /// Return the profile summary.
506 /// \c UseCS indicates whether to use the context-sensitive summary.
507 ProfileSummary &getSummary(bool UseCS) {
509 assert(CS_Summary && "No context sensitive summary");
510 return *(CS_Summary.get());
512 assert(Summary && "No profile summary");
513 return *(Summary.get());
518 } // end namespace llvm
520 #endif // LLVM_PROFILEDATA_INSTRPROFREADER_H