1 //===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains a pass that provides access to the global profile summary
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Analysis/ProfileSummaryInfo.h"
15 #include "llvm/Analysis/BlockFrequencyInfo.h"
16 #include "llvm/IR/BasicBlock.h"
17 #include "llvm/IR/CallSite.h"
18 #include "llvm/IR/Metadata.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/IR/ProfileSummary.h"
21 #include "llvm/InitializePasses.h"
22 #include "llvm/Support/CommandLine.h"
25 // The following two parameters determine the threshold for a count to be
26 // considered hot/cold. These two parameters are percentile values (multiplied
27 // by 10000). If the counts are sorted in descending order, the minimum count to
28 // reach ProfileSummaryCutoffHot gives the threshold to determine a hot count.
29 // Similarly, the minimum count to reach ProfileSummaryCutoffCold gives the
30 // threshold for determining cold count (everything <= this threshold is
33 static cl::opt<int> ProfileSummaryCutoffHot(
34 "profile-summary-cutoff-hot", cl::Hidden, cl::init(990000), cl::ZeroOrMore,
35 cl::desc("A count is hot if it exceeds the minimum count to"
36 " reach this percentile of total counts."));
38 static cl::opt<int> ProfileSummaryCutoffCold(
39 "profile-summary-cutoff-cold", cl::Hidden, cl::init(999999), cl::ZeroOrMore,
40 cl::desc("A count is cold if it is below the minimum count"
41 " to reach this percentile of total counts."));
43 static cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold(
44 "profile-summary-huge-working-set-size-threshold", cl::Hidden,
45 cl::init(15000), cl::ZeroOrMore,
46 cl::desc("The code working set size is considered huge if the number of"
47 " blocks required to reach the -profile-summary-cutoff-hot"
48 " percentile exceeds this count."));
50 static cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold(
51 "profile-summary-large-working-set-size-threshold", cl::Hidden,
52 cl::init(12500), cl::ZeroOrMore,
53 cl::desc("The code working set size is considered large if the number of"
54 " blocks required to reach the -profile-summary-cutoff-hot"
55 " percentile exceeds this count."));
57 // The next two options override the counts derived from summary computation and
58 // are useful for debugging purposes.
59 static cl::opt<int> ProfileSummaryHotCount(
60 "profile-summary-hot-count", cl::ReallyHidden, cl::ZeroOrMore,
61 cl::desc("A fixed hot count that overrides the count derived from"
62 " profile-summary-cutoff-hot"));
64 static cl::opt<int> ProfileSummaryColdCount(
65 "profile-summary-cold-count", cl::ReallyHidden, cl::ZeroOrMore,
66 cl::desc("A fixed cold count that overrides the count derived from"
67 " profile-summary-cutoff-cold"));
69 // Find the summary entry for a desired percentile of counts.
70 static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS,
71 uint64_t Percentile) {
72 auto It = partition_point(DS, [=](const ProfileSummaryEntry &Entry) {
73 return Entry.Cutoff < Percentile;
75 // The required percentile has to be <= one of the percentiles in the
78 report_fatal_error("Desired percentile exceeds the maximum cutoff");
82 // The profile summary metadata may be attached either by the frontend or by
83 // any backend passes (IR level instrumentation, for example). This method
84 // checks if the Summary is null and if so checks if the summary metadata is now
85 // available in the module and parses it to get the Summary object. Returns true
86 // if a valid Summary is available.
87 bool ProfileSummaryInfo::computeSummary() {
90 // First try to get context sensitive ProfileSummary.
91 auto *SummaryMD = M.getProfileSummary(/* IsCS */ true);
93 Summary.reset(ProfileSummary::getFromMD(SummaryMD));
96 // This will actually return PSK_Instr or PSK_Sample summary.
97 SummaryMD = M.getProfileSummary(/* IsCS */ false);
100 Summary.reset(ProfileSummary::getFromMD(SummaryMD));
105 ProfileSummaryInfo::getProfileCount(const Instruction *Inst,
106 BlockFrequencyInfo *BFI,
107 bool AllowSynthetic) {
110 assert((isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) &&
111 "We can only get profile count for call/invoke instruction.");
112 if (hasSampleProfile()) {
113 // In sample PGO mode, check if there is a profile metadata on the
114 // instruction. If it is present, determine hotness solely based on that,
115 // since the sampled entry count may not be accurate. If there is no
116 // annotated on the instruction, return None.
118 if (Inst->extractProfTotalWeight(TotalCount))
123 return BFI->getBlockProfileCount(Inst->getParent(), AllowSynthetic);
127 /// Returns true if the function's entry is hot. If it returns false, it
128 /// either means it is not hot or it is unknown whether it is hot or not (for
129 /// example, no profile data is available).
130 bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) {
131 if (!F || !computeSummary())
133 auto FunctionCount = F->getEntryCount();
134 // FIXME: The heuristic used below for determining hotness is based on
135 // preliminary SPEC tuning for inliner. This will eventually be a
136 // convenience method that calls isHotCount.
137 return FunctionCount && isHotCount(FunctionCount.getCount());
140 /// Returns true if the function contains hot code. This can include a hot
141 /// function entry count, hot basic block, or (in the case of Sample PGO)
142 /// hot total call edge count.
143 /// If it returns false, it either means it is not hot or it is unknown
144 /// (for example, no profile data is available).
145 bool ProfileSummaryInfo::isFunctionHotInCallGraph(const Function *F,
146 BlockFrequencyInfo &BFI) {
147 if (!F || !computeSummary())
149 if (auto FunctionCount = F->getEntryCount())
150 if (isHotCount(FunctionCount.getCount()))
153 if (hasSampleProfile()) {
154 uint64_t TotalCallCount = 0;
155 for (const auto &BB : *F)
156 for (const auto &I : BB)
157 if (isa<CallInst>(I) || isa<InvokeInst>(I))
158 if (auto CallCount = getProfileCount(&I, nullptr))
159 TotalCallCount += CallCount.getValue();
160 if (isHotCount(TotalCallCount))
163 for (const auto &BB : *F)
164 if (isHotBlock(&BB, &BFI))
169 /// Returns true if the function only contains cold code. This means that
170 /// the function entry and blocks are all cold, and (in the case of Sample PGO)
171 /// the total call edge count is cold.
172 /// If it returns false, it either means it is not cold or it is unknown
173 /// (for example, no profile data is available).
174 bool ProfileSummaryInfo::isFunctionColdInCallGraph(const Function *F,
175 BlockFrequencyInfo &BFI) {
176 if (!F || !computeSummary())
178 if (auto FunctionCount = F->getEntryCount())
179 if (!isColdCount(FunctionCount.getCount()))
182 if (hasSampleProfile()) {
183 uint64_t TotalCallCount = 0;
184 for (const auto &BB : *F)
185 for (const auto &I : BB)
186 if (isa<CallInst>(I) || isa<InvokeInst>(I))
187 if (auto CallCount = getProfileCount(&I, nullptr))
188 TotalCallCount += CallCount.getValue();
189 if (!isColdCount(TotalCallCount))
192 for (const auto &BB : *F)
193 if (!isColdBlock(&BB, &BFI))
198 // Like isFunctionHotInCallGraph but for a given cutoff.
199 bool ProfileSummaryInfo::isFunctionHotInCallGraphNthPercentile(
200 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) {
201 if (!F || !computeSummary())
203 if (auto FunctionCount = F->getEntryCount())
204 if (isHotCountNthPercentile(PercentileCutoff, FunctionCount.getCount()))
207 if (hasSampleProfile()) {
208 uint64_t TotalCallCount = 0;
209 for (const auto &BB : *F)
210 for (const auto &I : BB)
211 if (isa<CallInst>(I) || isa<InvokeInst>(I))
212 if (auto CallCount = getProfileCount(&I, nullptr))
213 TotalCallCount += CallCount.getValue();
214 if (isHotCountNthPercentile(PercentileCutoff, TotalCallCount))
217 for (const auto &BB : *F)
218 if (isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI))
223 /// Returns true if the function's entry is a cold. If it returns false, it
224 /// either means it is not cold or it is unknown whether it is cold or not (for
225 /// example, no profile data is available).
226 bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) {
229 if (F->hasFnAttribute(Attribute::Cold))
231 if (!computeSummary())
233 auto FunctionCount = F->getEntryCount();
234 // FIXME: The heuristic used below for determining coldness is based on
235 // preliminary SPEC tuning for inliner. This will eventually be a
236 // convenience method that calls isHotCount.
237 return FunctionCount && isColdCount(FunctionCount.getCount());
240 /// Compute the hot and cold thresholds.
241 void ProfileSummaryInfo::computeThresholds() {
242 if (!computeSummary())
244 auto &DetailedSummary = Summary->getDetailedSummary();
246 getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot);
247 HotCountThreshold = HotEntry.MinCount;
248 if (ProfileSummaryHotCount.getNumOccurrences() > 0)
249 HotCountThreshold = ProfileSummaryHotCount;
251 getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffCold);
252 ColdCountThreshold = ColdEntry.MinCount;
253 if (ProfileSummaryColdCount.getNumOccurrences() > 0)
254 ColdCountThreshold = ProfileSummaryColdCount;
255 assert(ColdCountThreshold <= HotCountThreshold &&
256 "Cold count threshold cannot exceed hot count threshold!");
257 HasHugeWorkingSetSize =
258 HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
259 HasLargeWorkingSetSize =
260 HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
263 Optional<uint64_t> ProfileSummaryInfo::computeThreshold(int PercentileCutoff) {
264 if (!computeSummary())
266 auto iter = ThresholdCache.find(PercentileCutoff);
267 if (iter != ThresholdCache.end()) {
270 auto &DetailedSummary = Summary->getDetailedSummary();
272 getEntryForPercentile(DetailedSummary, PercentileCutoff);
273 uint64_t CountThreshold = Entry.MinCount;
274 ThresholdCache[PercentileCutoff] = CountThreshold;
275 return CountThreshold;
278 bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
279 if (!HasHugeWorkingSetSize)
281 return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue();
284 bool ProfileSummaryInfo::hasLargeWorkingSetSize() {
285 if (!HasLargeWorkingSetSize)
287 return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue();
290 bool ProfileSummaryInfo::isHotCount(uint64_t C) {
291 if (!HotCountThreshold)
293 return HotCountThreshold && C >= HotCountThreshold.getValue();
296 bool ProfileSummaryInfo::isColdCount(uint64_t C) {
297 if (!ColdCountThreshold)
299 return ColdCountThreshold && C <= ColdCountThreshold.getValue();
302 bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, uint64_t C) {
303 auto CountThreshold = computeThreshold(PercentileCutoff);
304 return CountThreshold && C >= CountThreshold.getValue();
307 uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() {
308 if (!HotCountThreshold)
310 return HotCountThreshold ? HotCountThreshold.getValue() : UINT64_MAX;
313 uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() {
314 if (!ColdCountThreshold)
316 return ColdCountThreshold ? ColdCountThreshold.getValue() : 0;
319 bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) {
320 auto Count = BFI->getBlockProfileCount(BB);
321 return Count && isHotCount(*Count);
324 bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB,
325 BlockFrequencyInfo *BFI) {
326 auto Count = BFI->getBlockProfileCount(BB);
327 return Count && isColdCount(*Count);
330 bool ProfileSummaryInfo::isHotBlockNthPercentile(int PercentileCutoff,
331 const BasicBlock *BB,
332 BlockFrequencyInfo *BFI) {
333 auto Count = BFI->getBlockProfileCount(BB);
334 return Count && isHotCountNthPercentile(PercentileCutoff, *Count);
337 bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS,
338 BlockFrequencyInfo *BFI) {
339 auto C = getProfileCount(CS.getInstruction(), BFI);
340 return C && isHotCount(*C);
343 bool ProfileSummaryInfo::isColdCallSite(const CallSite &CS,
344 BlockFrequencyInfo *BFI) {
345 auto C = getProfileCount(CS.getInstruction(), BFI);
347 return isColdCount(*C);
349 // In SamplePGO, if the caller has been sampled, and there is no profile
350 // annotated on the callsite, we consider the callsite as cold.
351 return hasSampleProfile() && CS.getCaller()->hasProfileData();
354 INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",
355 "Profile summary info", false, true)
357 ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()
358 : ImmutablePass(ID) {
359 initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
362 bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {
363 PSI.reset(new ProfileSummaryInfo(M));
367 bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) {
372 AnalysisKey ProfileSummaryAnalysis::Key;
373 ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M,
374 ModuleAnalysisManager &) {
375 return ProfileSummaryInfo(M);
378 PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M,
379 ModuleAnalysisManager &AM) {
380 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
382 OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
385 if (PSI.isFunctionEntryHot(&F))
386 OS << " :hot entry ";
387 else if (PSI.isFunctionEntryCold(&F))
388 OS << " :cold entry ";
391 return PreservedAnalyses::all();
394 char ProfileSummaryInfoWrapperPass::ID = 0;