1 //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Instrumentation-based profiling data is generated by instrumented
11 // binaries through library functions in compiler-rt, and read by the clang
12 // frontend to feed PGO.
14 //===----------------------------------------------------------------------===//
16 #ifndef LLVM_PROFILEDATA_INSTRPROF_H
17 #define LLVM_PROFILEDATA_INSTRPROF_H
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ADT/StringSet.h"
23 #include "llvm/ADT/Triple.h"
24 #include "llvm/IR/GlobalValue.h"
25 #include "llvm/IR/ProfileSummary.h"
26 #include "llvm/ProfileData/InstrProfData.inc"
27 #include "llvm/Support/Compiler.h"
28 #include "llvm/Support/Endian.h"
29 #include "llvm/Support/Error.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/Host.h"
32 #include "llvm/Support/MD5.h"
33 #include "llvm/Support/MathExtras.h"
34 #include "llvm/Support/raw_ostream.h"
43 #include <system_error>
51 struct InstrProfRecord;
52 class InstrProfSymtab;
57 enum InstrProfSectKind {
58 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
59 #include "llvm/ProfileData/InstrProfData.inc"
62 /// Return the name of the profile section corresponding to \p IPSK.
64 /// The name of the section depends on the object format type \p OF. If
65 /// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
66 /// be added to the section name (this is the default).
67 std::string getInstrProfSectionName(InstrProfSectKind IPSK,
68 Triple::ObjectFormatType OF,
69 bool AddSegmentInfo = true);
71 /// Return the name profile runtime entry point to do value profiling
73 inline StringRef getInstrProfValueProfFuncName() {
74 return INSTR_PROF_VALUE_PROF_FUNC_STR;
77 /// Return the name profile runtime entry point to do value range profiling.
78 inline StringRef getInstrProfValueRangeProfFuncName() {
79 return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
82 /// Return the name prefix of variables containing instrumented function names.
83 inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
85 /// Return the name prefix of variables containing per-function control data.
86 inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
88 /// Return the name prefix of profile counter variables.
89 inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
91 /// Return the name prefix of value profile variables.
92 inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
94 /// Return the name of value profile node array variables:
95 inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
97 /// Return the name prefix of the COMDAT group for instrumentation variables
98 /// associated with a COMDAT function.
99 inline StringRef getInstrProfComdatPrefix() { return "__profv_"; }
101 /// Return the name of the variable holding the strings (possibly compressed)
102 /// of all function's PGO names.
103 inline StringRef getInstrProfNamesVarName() {
104 return "__llvm_prf_nm";
107 /// Return the name of a covarage mapping variable (internal linkage)
108 /// for each instrumented source module. Such variables are allocated
109 /// in the __llvm_covmap section.
110 inline StringRef getCoverageMappingVarName() {
111 return "__llvm_coverage_mapping";
114 /// Return the name of the internal variable recording the array
115 /// of PGO name vars referenced by the coverage mapping. The owning
116 /// functions of those names are not emitted by FE (e.g, unused inline
118 inline StringRef getCoverageUnusedNamesVarName() {
119 return "__llvm_coverage_names";
122 /// Return the name of function that registers all the per-function control
123 /// data at program startup time by calling __llvm_register_function. This
124 /// function has internal linkage and is called by __llvm_profile_init
125 /// runtime method. This function is not generated for these platforms:
126 /// Darwin, Linux, and FreeBSD.
127 inline StringRef getInstrProfRegFuncsName() {
128 return "__llvm_profile_register_functions";
131 /// Return the name of the runtime interface that registers per-function control
132 /// data for one instrumented function.
133 inline StringRef getInstrProfRegFuncName() {
134 return "__llvm_profile_register_function";
137 /// Return the name of the runtime interface that registers the PGO name strings.
138 inline StringRef getInstrProfNamesRegFuncName() {
139 return "__llvm_profile_register_names_function";
142 /// Return the name of the runtime initialization method that is generated by
143 /// the compiler. The function calls __llvm_profile_register_functions and
144 /// __llvm_profile_override_default_filename functions if needed. This function
145 /// has internal linkage and invoked at startup time via init_array.
146 inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
148 /// Return the name of the hook variable defined in profile runtime library.
149 /// A reference to the variable causes the linker to link in the runtime
150 /// initialization module (which defines the hook variable).
151 inline StringRef getInstrProfRuntimeHookVarName() {
152 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
155 /// Return the name of the compiler generated function that references the
156 /// runtime hook variable. The function is a weak global.
157 inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
158 return "__llvm_profile_runtime_user";
161 /// Return the marker used to separate PGO names during serialization.
162 inline StringRef getInstrProfNameSeparator() { return "\01"; }
164 /// Return the modified name for function \c F suitable to be
165 /// used the key for profile lookup. Variable \c InLTO indicates if this
166 /// is called in LTO optimization passes.
167 std::string getPGOFuncName(const Function &F, bool InLTO = false,
168 uint64_t Version = INSTR_PROF_INDEX_VERSION);
170 /// Return the modified name for a function suitable to be
171 /// used the key for profile lookup. The function's original
172 /// name is \c RawFuncName and has linkage of type \c Linkage.
173 /// The function is defined in module \c FileName.
174 std::string getPGOFuncName(StringRef RawFuncName,
175 GlobalValue::LinkageTypes Linkage,
177 uint64_t Version = INSTR_PROF_INDEX_VERSION);
179 /// Return the name of the global variable used to store a function
180 /// name in PGO instrumentation. \c FuncName is the name of the function
181 /// returned by the \c getPGOFuncName call.
182 std::string getPGOFuncNameVarName(StringRef FuncName,
183 GlobalValue::LinkageTypes Linkage);
185 /// Create and return the global variable for function name used in PGO
186 /// instrumentation. \c FuncName is the name of the function returned
187 /// by \c getPGOFuncName call.
188 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
190 /// Create and return the global variable for function name used in PGO
191 /// instrumentation. /// \c FuncName is the name of the function
192 /// returned by \c getPGOFuncName call, \c M is the owning module,
193 /// and \c Linkage is the linkage of the instrumented function.
194 GlobalVariable *createPGOFuncNameVar(Module &M,
195 GlobalValue::LinkageTypes Linkage,
196 StringRef PGOFuncName);
198 /// Return the initializer in string of the PGO name var \c NameVar.
199 StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
201 /// Given a PGO function name, remove the filename prefix and return
202 /// the original (static) function name.
203 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
204 StringRef FileName = "<unknown>");
206 /// Given a vector of strings (function PGO names) \c NameStrs, the
207 /// method generates a combined string \c Result thatis ready to be
208 /// serialized. The \c Result string is comprised of three fields:
209 /// The first field is the legnth of the uncompressed strings, and the
210 /// the second field is the length of the zlib-compressed string.
211 /// Both fields are encoded in ULEB128. If \c doCompress is false, the
212 /// third field is the uncompressed strings; otherwise it is the
213 /// compressed string. When the string compression is off, the
214 /// second field will have value zero.
215 Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
216 bool doCompression, std::string &Result);
218 /// Produce \c Result string with the same format described above. The input
219 /// is vector of PGO function name variables that are referenced.
220 Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
221 std::string &Result, bool doCompression = true);
223 /// \c NameStrings is a string composed of one of more sub-strings encoded in
224 /// the format described above. The substrings are separated by 0 or more zero
225 /// bytes. This method decodes the string and populates the \c Symtab.
226 Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
228 /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
229 /// set in IR PGO compilation.
230 bool isIRPGOFlagSet(const Module *M);
232 /// Check if we can safely rename this Comdat function. Instances of the same
233 /// comdat function may have different control flows thus can not share the
234 /// same counter variable.
235 bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
237 enum InstrProfValueKind : uint32_t {
238 #define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value,
239 #include "llvm/ProfileData/InstrProfData.inc"
242 /// Get the value profile data for value site \p SiteIdx from \p InstrProfR
243 /// and annotate the instruction \p Inst with the value profile meta data.
244 /// Annotate up to \p MaxMDCount (default 3) number of records per value site.
245 void annotateValueSite(Module &M, Instruction &Inst,
246 const InstrProfRecord &InstrProfR,
247 InstrProfValueKind ValueKind, uint32_t SiteIndx,
248 uint32_t MaxMDCount = 3);
250 /// Same as the above interface but using an ArrayRef, as well as \p Sum.
251 void annotateValueSite(Module &M, Instruction &Inst,
252 ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
253 InstrProfValueKind ValueKind, uint32_t MaxMDCount);
255 /// Extract the value profile data from \p Inst which is annotated with
256 /// value profile meta data. Return false if there is no value data annotated,
257 /// otherwise return true.
258 bool getValueProfDataFromInst(const Instruction &Inst,
259 InstrProfValueKind ValueKind,
260 uint32_t MaxNumValueData,
261 InstrProfValueData ValueData[],
262 uint32_t &ActualNumValueData, uint64_t &TotalC);
264 inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
266 /// Return the PGOFuncName meta data associated with a function.
267 MDNode *getPGOFuncNameMetadata(const Function &F);
269 /// Create the PGOFuncName meta data if PGOFuncName is different from
270 /// function's raw name. This should only apply to internal linkage functions
271 /// declared by users only.
272 void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
274 /// Check if we can use Comdat for profile variables. This will eliminate
275 /// the duplicated profile variables for Comdat functions.
276 bool needsComdatForCounter(const Function &F, const Module &M);
278 const std::error_category &instrprof_category();
280 enum class instrprof_error {
287 unsupported_hash_type,
295 value_site_count_mismatch,
301 inline std::error_code make_error_code(instrprof_error E) {
302 return std::error_code(static_cast<int>(E), instrprof_category());
305 class InstrProfError : public ErrorInfo<InstrProfError> {
307 InstrProfError(instrprof_error Err) : Err(Err) {
308 assert(Err != instrprof_error::success && "Not an error");
311 std::string message() const override;
313 void log(raw_ostream &OS) const override { OS << message(); }
315 std::error_code convertToErrorCode() const override {
316 return make_error_code(Err);
319 instrprof_error get() const { return Err; }
321 /// Consume an Error and return the raw enum value contained within it. The
322 /// Error must either be a success value, or contain a single InstrProfError.
323 static instrprof_error take(Error E) {
324 auto Err = instrprof_error::success;
325 handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) {
326 assert(Err == instrprof_error::success && "Multiple errors encountered");
338 class SoftInstrProfErrors {
339 /// Count the number of soft instrprof_errors encountered and keep track of
340 /// the first such error for reporting purposes.
342 /// The first soft error encountered.
343 instrprof_error FirstError = instrprof_error::success;
345 /// The number of hash mismatches.
346 unsigned NumHashMismatches = 0;
348 /// The number of count mismatches.
349 unsigned NumCountMismatches = 0;
351 /// The number of counter overflows.
352 unsigned NumCounterOverflows = 0;
354 /// The number of value site count mismatches.
355 unsigned NumValueSiteCountMismatches = 0;
358 SoftInstrProfErrors() = default;
360 ~SoftInstrProfErrors() {
361 assert(FirstError == instrprof_error::success &&
362 "Unchecked soft error encountered");
365 /// Track a soft error (\p IE) and increment its associated counter.
366 void addError(instrprof_error IE);
368 /// Get the number of hash mismatches.
369 unsigned getNumHashMismatches() const { return NumHashMismatches; }
371 /// Get the number of count mismatches.
372 unsigned getNumCountMismatches() const { return NumCountMismatches; }
374 /// Get the number of counter overflows.
375 unsigned getNumCounterOverflows() const { return NumCounterOverflows; }
377 /// Get the number of value site count mismatches.
378 unsigned getNumValueSiteCountMismatches() const {
379 return NumValueSiteCountMismatches;
382 /// Return the first encountered error and reset FirstError to a success
385 if (FirstError == instrprof_error::success)
386 return Error::success();
387 auto E = make_error<InstrProfError>(FirstError);
388 FirstError = instrprof_error::success;
397 } // end namespace object
399 namespace IndexedInstrProf {
401 uint64_t ComputeHash(StringRef K);
403 } // end namespace IndexedInstrProf
405 /// A symbol table used for function PGO name look-up with keys
406 /// (such as pointers, md5hash values) to the function. A function's
407 /// PGO name or name's md5hash are used in retrieving the profile
408 /// data of the function. See \c getPGOFuncName() method for details
409 /// on how PGO name is formed.
410 class InstrProfSymtab {
412 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
416 uint64_t Address = 0;
417 // Unique name strings.
419 // A map from MD5 keys to function name strings.
420 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
421 // A map from MD5 keys to function define. We only populate this map
422 // when build the Symtab from a Module.
423 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
424 // A map from function runtime address to function name MD5 hash.
425 // This map is only populated and used by raw instr profile reader.
426 AddrHashMap AddrToMD5Map;
429 InstrProfSymtab() = default;
431 /// Create InstrProfSymtab from an object file section which
432 /// contains function PGO names. When section may contain raw
433 /// string data or string data in compressed form. This method
434 /// only initialize the symtab with reference to the data and
435 /// the section base address. The decompression will be delayed
436 /// until before it is used. See also \c create(StringRef) method.
437 Error create(object::SectionRef &Section);
439 /// This interface is used by reader of CoverageMapping test
441 inline Error create(StringRef D, uint64_t BaseAddr);
443 /// \c NameStrings is a string composed of one of more sub-strings
444 /// encoded in the format described in \c collectPGOFuncNameStrings.
445 /// This method is a wrapper to \c readPGOFuncNameStrings method.
446 inline Error create(StringRef NameStrings);
448 /// A wrapper interface to populate the PGO symtab with functions
449 /// decls from module \c M. This interface is used by transformation
450 /// passes such as indirect function call promotion. Variable \c InLTO
451 /// indicates if this is called from LTO optimization passes.
452 Error create(Module &M, bool InLTO = false);
454 /// Create InstrProfSymtab from a set of names iteratable from
455 /// \p IterRange. This interface is used by IndexedProfReader.
456 template <typename NameIterRange> Error create(const NameIterRange &IterRange);
458 // If the symtab is created by a series of calls to \c addFuncName, \c
459 // finalizeSymtab needs to be called before looking up function names.
460 // This is required because the underlying map is a vector (for space
461 // efficiency) which needs to be sorted.
462 inline void finalizeSymtab();
464 /// Update the symtab by adding \p FuncName to the table. This interface
465 /// is used by the raw and text profile readers.
466 Error addFuncName(StringRef FuncName) {
467 if (FuncName.empty())
468 return make_error<InstrProfError>(instrprof_error::malformed);
469 auto Ins = NameTab.insert(FuncName);
471 MD5NameMap.push_back(std::make_pair(
472 IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
473 return Error::success();
476 /// Map a function address to its name's MD5 hash. This interface
477 /// is only used by the raw profiler reader.
478 void mapAddress(uint64_t Addr, uint64_t MD5Val) {
479 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
482 AddrHashMap &getAddrHashMap() { return AddrToMD5Map; }
484 /// Return function's PGO name from the function name's symbol
485 /// address in the object file. If an error occurs, return
487 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
489 /// Return function's PGO name from the name's md5 hash value.
490 /// If not found, return an empty string.
491 inline StringRef getFuncName(uint64_t FuncMD5Hash);
493 /// Return function from the name's md5 hash. Return nullptr if not found.
494 inline Function *getFunction(uint64_t FuncMD5Hash);
496 /// Return the function's original assembly name by stripping off
497 /// the prefix attached (to symbols with priviate linkage). For
498 /// global functions, it returns the same string as getFuncName.
499 inline StringRef getOrigFuncName(uint64_t FuncMD5Hash);
501 /// Return the name section data.
502 inline StringRef getNameData() const { return Data; }
505 Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
508 return Error::success();
511 Error InstrProfSymtab::create(StringRef NameStrings) {
512 return readPGOFuncNameStrings(NameStrings, *this);
515 template <typename NameIterRange>
516 Error InstrProfSymtab::create(const NameIterRange &IterRange) {
517 for (auto Name : IterRange)
518 if (Error E = addFuncName(Name))
522 return Error::success();
525 void InstrProfSymtab::finalizeSymtab() {
526 std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first());
527 std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first());
528 std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first());
529 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
533 StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) {
535 std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash,
536 [](const std::pair<uint64_t, std::string> &LHS,
537 uint64_t RHS) { return LHS.first < RHS; });
538 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
539 return Result->second;
543 Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
545 std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash,
546 [](const std::pair<uint64_t, Function*> &LHS,
547 uint64_t RHS) { return LHS.first < RHS; });
548 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
549 return Result->second;
553 // See also getPGOFuncName implementation. These two need to be
555 StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) {
556 StringRef PGOName = getFuncName(FuncMD5Hash);
557 size_t S = PGOName.find_first_of(':');
558 if (S == StringRef::npos)
560 return PGOName.drop_front(S + 1);
563 struct InstrProfValueSiteRecord {
564 /// Value profiling data pairs at a given value site.
565 std::list<InstrProfValueData> ValueData;
567 InstrProfValueSiteRecord() { ValueData.clear(); }
568 template <class InputIterator>
569 InstrProfValueSiteRecord(InputIterator F, InputIterator L)
572 /// Sort ValueData ascending by Value
573 void sortByTargetValues() {
575 [](const InstrProfValueData &left, const InstrProfValueData &right) {
576 return left.Value < right.Value;
579 /// Sort ValueData Descending by Count
580 inline void sortByCount();
582 /// Merge data from another InstrProfValueSiteRecord
583 /// Optionally scale merged counts by \p Weight.
584 void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
585 function_ref<void(instrprof_error)> Warn);
586 /// Scale up value profile data counts.
587 void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
590 /// Profiling information for a single function.
591 struct InstrProfRecord {
592 std::vector<uint64_t> Counts;
594 InstrProfRecord() = default;
595 InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
596 InstrProfRecord(InstrProfRecord &&) = default;
597 InstrProfRecord(const InstrProfRecord &RHS)
598 : Counts(RHS.Counts),
599 ValueData(RHS.ValueData
600 ? llvm::make_unique<ValueProfData>(*RHS.ValueData)
602 InstrProfRecord &operator=(InstrProfRecord &&) = default;
603 InstrProfRecord &operator=(const InstrProfRecord &RHS) {
605 if (!RHS.ValueData) {
610 ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData);
612 *ValueData = *RHS.ValueData;
616 using ValueMapType = std::vector<std::pair<uint64_t, uint64_t>>;
618 /// Return the number of value profile kinds with non-zero number
619 /// of profile sites.
620 inline uint32_t getNumValueKinds() const;
621 /// Return the number of instrumented sites for ValueKind.
622 inline uint32_t getNumValueSites(uint32_t ValueKind) const;
624 /// Return the total number of ValueData for ValueKind.
625 inline uint32_t getNumValueData(uint32_t ValueKind) const;
627 /// Return the number of value data collected for ValueKind at profiling
629 inline uint32_t getNumValueDataForSite(uint32_t ValueKind,
630 uint32_t Site) const;
632 /// Return the array of profiled values at \p Site. If \p TotalC
633 /// is not null, the total count of all target values at this site
634 /// will be stored in \c *TotalC.
635 inline std::unique_ptr<InstrProfValueData[]>
636 getValueForSite(uint32_t ValueKind, uint32_t Site,
637 uint64_t *TotalC = nullptr) const;
639 /// Get the target value/counts of kind \p ValueKind collected at site
640 /// \p Site and store the result in array \p Dest. Return the total
641 /// counts of all target values at this site.
642 inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
643 uint32_t Site) const;
645 /// Reserve space for NumValueSites sites.
646 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
648 /// Add ValueData for ValueKind at value Site.
649 void addValueData(uint32_t ValueKind, uint32_t Site,
650 InstrProfValueData *VData, uint32_t N,
651 ValueMapType *ValueMap);
653 /// Merge the counts in \p Other into this one.
654 /// Optionally scale merged counts by \p Weight.
655 void merge(InstrProfRecord &Other, uint64_t Weight,
656 function_ref<void(instrprof_error)> Warn);
658 /// Scale up profile counts (including value profile data) by
660 void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
662 /// Sort value profile data (per site) by count.
663 void sortValueData() {
664 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
665 for (auto &SR : getValueSitesForKind(Kind))
669 /// Clear value data entries and edge counters.
675 /// Clear value data entries
676 void clearValueData() { ValueData = nullptr; }
679 struct ValueProfData {
680 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
681 std::vector<InstrProfValueSiteRecord> MemOPSizes;
683 std::unique_ptr<ValueProfData> ValueData;
685 MutableArrayRef<InstrProfValueSiteRecord>
686 getValueSitesForKind(uint32_t ValueKind) {
687 // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
688 // implemented in LLVM) to call the const overload of this function, then
689 // cast away the constness from the result.
690 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
692 return makeMutableArrayRef(
693 const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
695 ArrayRef<InstrProfValueSiteRecord>
696 getValueSitesForKind(uint32_t ValueKind) const {
700 case IPVK_IndirectCallTarget:
701 return ValueData->IndirectCallSites;
703 return ValueData->MemOPSizes;
705 llvm_unreachable("Unknown value kind!");
709 std::vector<InstrProfValueSiteRecord> &
710 getOrCreateValueSitesForKind(uint32_t ValueKind) {
712 ValueData = llvm::make_unique<ValueProfData>();
714 case IPVK_IndirectCallTarget:
715 return ValueData->IndirectCallSites;
717 return ValueData->MemOPSizes;
719 llvm_unreachable("Unknown value kind!");
723 // Map indirect call target name hash to name string.
724 uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
725 ValueMapType *HashKeys);
727 // Merge Value Profile data from Src record to this record for ValueKind.
728 // Scale merged value counts by \p Weight.
729 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
731 function_ref<void(instrprof_error)> Warn);
733 // Scale up value profile data count.
734 void scaleValueProfData(uint32_t ValueKind, uint64_t Weight,
735 function_ref<void(instrprof_error)> Warn);
738 struct NamedInstrProfRecord : InstrProfRecord {
742 NamedInstrProfRecord() = default;
743 NamedInstrProfRecord(StringRef Name, uint64_t Hash,
744 std::vector<uint64_t> Counts)
745 : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
748 uint32_t InstrProfRecord::getNumValueKinds() const {
749 uint32_t NumValueKinds = 0;
750 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
751 NumValueKinds += !(getValueSitesForKind(Kind).empty());
752 return NumValueKinds;
755 uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
757 for (auto &SR : getValueSitesForKind(ValueKind))
758 N += SR.ValueData.size();
762 uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
763 return getValueSitesForKind(ValueKind).size();
766 uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind,
767 uint32_t Site) const {
768 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
771 std::unique_ptr<InstrProfValueData[]>
772 InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site,
773 uint64_t *TotalC) const {
775 uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC);
776 uint32_t N = getNumValueDataForSite(ValueKind, Site);
779 return std::unique_ptr<InstrProfValueData[]>(nullptr);
782 auto VD = llvm::make_unique<InstrProfValueData[]>(N);
783 TotalCount = getValueForSite(VD.get(), ValueKind, Site);
788 uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[],
790 uint32_t Site) const {
792 uint64_t TotalCount = 0;
793 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
794 Dest[I].Value = V.Value;
795 Dest[I].Count = V.Count;
796 TotalCount = SaturatingAdd(TotalCount, V.Count);
802 void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
805 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
808 inline support::endianness getHostEndianness() {
809 return sys::IsLittleEndianHost ? support::little : support::big;
812 // Include definitions for value profile data
813 #define INSTR_PROF_VALUE_PROF_DATA
814 #include "llvm/ProfileData/InstrProfData.inc"
816 void InstrProfValueSiteRecord::sortByCount() {
818 [](const InstrProfValueData &left, const InstrProfValueData &right) {
819 return left.Count > right.Count;
822 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
823 if (ValueData.size() > max_s)
824 ValueData.resize(max_s);
827 namespace IndexedInstrProf {
829 enum class HashT : uint32_t {
834 inline uint64_t ComputeHash(HashT Type, StringRef K) {
839 llvm_unreachable("Unhandled hash type");
842 const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
845 // Version 1 is the first version. In this version, the value of
846 // a key/value pair can only include profile data of a single function.
847 // Due to this restriction, the number of block counters for a given
848 // function is not recorded but derived from the length of the value.
850 // The version 2 format supports recording profile data of multiple
851 // functions which share the same key in one value field. To support this,
852 // the number block counters is recorded as an uint64_t field right after the
853 // function structural hash.
855 // Version 3 supports value profile data. The value profile data is expected
856 // to follow the block counter profile data.
858 // In this version, profile summary data \c IndexedInstrProf::Summary is
859 // stored after the profile header.
861 // The current version is 4.
862 CurrentVersion = INSTR_PROF_INDEX_VERSION
864 const uint64_t Version = ProfVersion::CurrentVersion;
866 const HashT HashType = HashT::MD5;
868 inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); }
870 // This structure defines the file header of the LLVM profile
871 // data file in indexed-format.
875 uint64_t Unused; // Becomes unused since version 4
880 // Profile summary data recorded in the profile data file in indexed
881 // format. It is introduced in version 4. The summary data follows
882 // right after the profile file header.
885 uint64_t Cutoff; ///< The required percentile of total execution count.
887 MinBlockCount; ///< The minimum execution count for this percentile.
888 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
890 // The field kind enumerator to assigned value mapping should remain
891 // unchanged when a new kind is added or an old kind gets deleted in
893 enum SummaryFieldKind {
894 /// The total number of functions instrumented.
895 TotalNumFunctions = 0,
896 /// Total number of instrumented blocks/edges.
898 /// The maximal execution count among all functions.
899 /// This field does not exist for profile data from IR based
901 MaxFunctionCount = 2,
902 /// Max block count of the program.
904 /// Max internal block count of the program (excluding entry blocks).
905 MaxInternalBlockCount = 4,
906 /// The sum of all instrumented block counts.
908 NumKinds = TotalBlockCount + 1
911 // The number of summmary fields following the summary header.
912 uint64_t NumSummaryFields;
913 // The number of Cutoff Entries (Summary::Entry) following summary fields.
914 uint64_t NumCutoffEntries;
917 Summary(uint32_t Size) { memset(this, 0, Size); }
919 void operator delete(void *ptr) { ::operator delete(ptr); }
921 static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) {
922 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
923 NumSumFields * sizeof(uint64_t);
926 const uint64_t *getSummaryDataBase() const {
927 return reinterpret_cast<const uint64_t *>(this + 1);
930 uint64_t *getSummaryDataBase() {
931 return reinterpret_cast<uint64_t *>(this + 1);
934 const Entry *getCutoffEntryBase() const {
935 return reinterpret_cast<const Entry *>(
936 &getSummaryDataBase()[NumSummaryFields]);
939 Entry *getCutoffEntryBase() {
940 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
943 uint64_t get(SummaryFieldKind K) const {
944 return getSummaryDataBase()[K];
947 void set(SummaryFieldKind K, uint64_t V) {
948 getSummaryDataBase()[K] = V;
951 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
953 void setEntry(uint32_t I, const ProfileSummaryEntry &E) {
954 Entry &ER = getCutoffEntryBase()[I];
955 ER.Cutoff = E.Cutoff;
956 ER.MinBlockCount = E.MinCount;
957 ER.NumBlocks = E.NumCounts;
961 inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
962 return std::unique_ptr<Summary>(new (::operator new(TotalSize))
966 } // end namespace IndexedInstrProf
968 namespace RawInstrProf {
970 // Version 1: First version
971 // Version 2: Added value profile data section. Per-function control data
972 // struct has more fields to describe value profile information.
973 // Version 3: Compressed name section support. Function PGO name reference
974 // from control data struct is changed from raw pointer to Name's MD5 value.
975 // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
977 const uint64_t Version = INSTR_PROF_RAW_VERSION;
979 template <class IntPtrT> inline uint64_t getMagic();
980 template <> inline uint64_t getMagic<uint64_t>() {
981 return INSTR_PROF_RAW_MAGIC_64;
984 template <> inline uint64_t getMagic<uint32_t>() {
985 return INSTR_PROF_RAW_MAGIC_32;
988 // Per-function profile data header/control structure.
989 // The definition should match the structure defined in
990 // compiler-rt/lib/profile/InstrProfiling.h.
991 // It should also match the synthesized type in
992 // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
993 template <class IntPtrT> struct LLVM_ALIGNAS(8) ProfileData {
994 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
995 #include "llvm/ProfileData/InstrProfData.inc"
998 // File header structure of the LLVM profile data in raw format.
999 // The definition should match the header referenced in
1000 // compiler-rt/lib/profile/InstrProfilingFile.c and
1001 // InstrProfilingBuffer.c.
1003 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1004 #include "llvm/ProfileData/InstrProfData.inc"
1007 } // end namespace RawInstrProf
1009 // Parse MemOP Size range option.
1010 void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
1011 int64_t &RangeLast);
1013 } // end namespace llvm
1015 #endif // LLVM_PROFILEDATA_INSTRPROF_H