1 //===- CallGraphSCCPass.cpp - Pass that operates BU on call graph ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the CallGraphSCCPass class, which is used for passes
11 // which are implemented as bottom-up traversals on the call graph. Because
12 // there may be cycles in the call graph, passes of this type operate on the
13 // call-graph in SCC order: that is, they process function bottom-up, except for
14 // recursive functions, which they process all at once.
16 //===----------------------------------------------------------------------===//
18 #include "llvm/Analysis/CallGraphSCCPass.h"
19 #include "llvm/ADT/DenseMap.h"
20 #include "llvm/ADT/SCCIterator.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/CallGraph.h"
23 #include "llvm/IR/CallSite.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/IR/LegacyPassManagers.h"
28 #include "llvm/IR/Module.h"
29 #include "llvm/IR/OptBisect.h"
30 #include "llvm/Pass.h"
31 #include "llvm/Support/CommandLine.h"
32 #include "llvm/Support/Debug.h"
33 #include "llvm/Support/Timer.h"
34 #include "llvm/Support/raw_ostream.h"
42 #define DEBUG_TYPE "cgscc-passmgr"
44 static cl::opt<unsigned>
45 MaxIterations("max-cg-scc-iterations", cl::ReallyHidden, cl::init(4));
47 STATISTIC(MaxSCCIterations, "Maximum CGSCCPassMgr iterations on one SCC");
49 //===----------------------------------------------------------------------===//
52 /// CGPassManager manages FPPassManagers and CallGraphSCCPasses.
56 class CGPassManager : public ModulePass, public PMDataManager {
60 explicit CGPassManager() : ModulePass(ID), PMDataManager() {}
62 /// Execute all of the passes scheduled for execution. Keep track of
63 /// whether any of the passes modifies the module, and if so, return true.
64 bool runOnModule(Module &M) override;
66 using ModulePass::doInitialization;
67 using ModulePass::doFinalization;
69 bool doInitialization(CallGraph &CG);
70 bool doFinalization(CallGraph &CG);
72 /// Pass Manager itself does not invalidate any analysis info.
73 void getAnalysisUsage(AnalysisUsage &Info) const override {
74 // CGPassManager walks SCC and it needs CallGraph.
75 Info.addRequired<CallGraphWrapperPass>();
76 Info.setPreservesAll();
79 StringRef getPassName() const override { return "CallGraph Pass Manager"; }
81 PMDataManager *getAsPMDataManager() override { return this; }
82 Pass *getAsPass() override { return this; }
84 // Print passes managed by this manager
85 void dumpPassStructure(unsigned Offset) override {
86 errs().indent(Offset*2) << "Call Graph SCC Pass Manager\n";
87 for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
88 Pass *P = getContainedPass(Index);
89 P->dumpPassStructure(Offset + 1);
90 dumpLastUses(P, Offset+1);
94 Pass *getContainedPass(unsigned N) {
95 assert(N < PassVector.size() && "Pass number out of range!");
96 return static_cast<Pass *>(PassVector[N]);
99 PassManagerType getPassManagerType() const override {
100 return PMT_CallGraphPassManager;
104 bool RunAllPassesOnSCC(CallGraphSCC &CurSCC, CallGraph &CG,
105 bool &DevirtualizedCall);
107 bool RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC,
108 CallGraph &CG, bool &CallGraphUpToDate,
109 bool &DevirtualizedCall);
110 bool RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG,
111 bool IsCheckingMode);
114 } // end anonymous namespace.
116 char CGPassManager::ID = 0;
118 bool CGPassManager::RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC,
119 CallGraph &CG, bool &CallGraphUpToDate,
120 bool &DevirtualizedCall) {
121 bool Changed = false;
122 PMDataManager *PM = P->getAsPMDataManager();
125 CallGraphSCCPass *CGSP = (CallGraphSCCPass*)P;
126 if (!CallGraphUpToDate) {
127 DevirtualizedCall |= RefreshCallGraph(CurSCC, CG, false);
128 CallGraphUpToDate = true;
132 TimeRegion PassTimer(getPassTimer(CGSP));
133 Changed = CGSP->runOnSCC(CurSCC);
136 // After the CGSCCPass is done, when assertions are enabled, use
137 // RefreshCallGraph to verify that the callgraph was correctly updated.
140 RefreshCallGraph(CurSCC, CG, true);
146 assert(PM->getPassManagerType() == PMT_FunctionPassManager &&
147 "Invalid CGPassManager member");
148 FPPassManager *FPP = (FPPassManager*)P;
150 // Run pass P on all functions in the current SCC.
151 for (CallGraphNode *CGN : CurSCC) {
152 if (Function *F = CGN->getFunction()) {
153 dumpPassInfo(P, EXECUTION_MSG, ON_FUNCTION_MSG, F->getName());
155 TimeRegion PassTimer(getPassTimer(FPP));
156 Changed |= FPP->runOnFunction(*F);
158 F->getContext().yield();
162 // The function pass(es) modified the IR, they may have clobbered the
164 if (Changed && CallGraphUpToDate) {
165 DEBUG(dbgs() << "CGSCCPASSMGR: Pass Dirtied SCC: "
166 << P->getPassName() << '\n');
167 CallGraphUpToDate = false;
172 /// Scan the functions in the specified CFG and resync the
173 /// callgraph with the call sites found in it. This is used after
174 /// FunctionPasses have potentially munged the callgraph, and can be used after
175 /// CallGraphSCC passes to verify that they correctly updated the callgraph.
177 /// This function returns true if it devirtualized an existing function call,
178 /// meaning it turned an indirect call into a direct call. This happens when
179 /// a function pass like GVN optimizes away stuff feeding the indirect call.
180 /// This never happens in checking mode.
181 bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG,
183 DenseMap<Value*, CallGraphNode*> CallSites;
185 DEBUG(dbgs() << "CGSCCPASSMGR: Refreshing SCC with " << CurSCC.size()
187 for (CallGraphNode *CGN : CurSCC)
191 bool MadeChange = false;
192 bool DevirtualizedCall = false;
194 // Scan all functions in the SCC.
195 unsigned FunctionNo = 0;
196 for (CallGraphSCC::iterator SCCIdx = CurSCC.begin(), E = CurSCC.end();
197 SCCIdx != E; ++SCCIdx, ++FunctionNo) {
198 CallGraphNode *CGN = *SCCIdx;
199 Function *F = CGN->getFunction();
200 if (!F || F->isDeclaration()) continue;
202 // Walk the function body looking for call sites. Sync up the call sites in
203 // CGN with those actually in the function.
205 // Keep track of the number of direct and indirect calls that were
206 // invalidated and removed.
207 unsigned NumDirectRemoved = 0, NumIndirectRemoved = 0;
209 // Get the set of call sites currently in the function.
210 for (CallGraphNode::iterator I = CGN->begin(), E = CGN->end(); I != E; ) {
211 // If this call site is null, then the function pass deleted the call
212 // entirely and the WeakTrackingVH nulled it out.
214 // If we've already seen this call site, then the FunctionPass RAUW'd
215 // one call with another, which resulted in two "uses" in the edge
216 // list of the same call.
217 CallSites.count(I->first) ||
219 // If the call edge is not from a call or invoke, or it is a
220 // instrinsic call, then the function pass RAUW'd a call with
221 // another value. This can happen when constant folding happens
222 // of well known functions etc.
223 !CallSite(I->first) ||
224 (CallSite(I->first).getCalledFunction() &&
225 CallSite(I->first).getCalledFunction()->isIntrinsic() &&
227 CallSite(I->first).getCalledFunction()->getIntrinsicID()))) {
228 assert(!CheckingMode &&
229 "CallGraphSCCPass did not update the CallGraph correctly!");
231 // If this was an indirect call site, count it.
232 if (!I->second->getFunction())
233 ++NumIndirectRemoved;
237 // Just remove the edge from the set of callees, keep track of whether
238 // I points to the last element of the vector.
239 bool WasLast = I + 1 == E;
240 CGN->removeCallEdge(I);
242 // If I pointed to the last element of the vector, we have to bail out:
243 // iterator checking rejects comparisons of the resultant pointer with
251 assert(!CallSites.count(I->first) &&
252 "Call site occurs in node multiple times");
254 CallSite CS(I->first);
256 Function *Callee = CS.getCalledFunction();
257 // Ignore intrinsics because they're not really function calls.
258 if (!Callee || !(Callee->isIntrinsic()))
259 CallSites.insert(std::make_pair(I->first, I->second));
264 // Loop over all of the instructions in the function, getting the callsites.
265 // Keep track of the number of direct/indirect calls added.
266 unsigned NumDirectAdded = 0, NumIndirectAdded = 0;
268 for (BasicBlock &BB : *F)
269 for (Instruction &I : BB) {
272 Function *Callee = CS.getCalledFunction();
273 if (Callee && Callee->isIntrinsic()) continue;
275 // If this call site already existed in the callgraph, just verify it
276 // matches up to expectations and remove it from CallSites.
277 DenseMap<Value*, CallGraphNode*>::iterator ExistingIt =
278 CallSites.find(CS.getInstruction());
279 if (ExistingIt != CallSites.end()) {
280 CallGraphNode *ExistingNode = ExistingIt->second;
282 // Remove from CallSites since we have now seen it.
283 CallSites.erase(ExistingIt);
285 // Verify that the callee is right.
286 if (ExistingNode->getFunction() == CS.getCalledFunction())
289 // If we are in checking mode, we are not allowed to actually mutate
290 // the callgraph. If this is a case where we can infer that the
291 // callgraph is less precise than it could be (e.g. an indirect call
292 // site could be turned direct), don't reject it in checking mode, and
293 // don't tweak it to be more precise.
294 if (CheckingMode && CS.getCalledFunction() &&
295 ExistingNode->getFunction() == nullptr)
298 assert(!CheckingMode &&
299 "CallGraphSCCPass did not update the CallGraph correctly!");
301 // If not, we either went from a direct call to indirect, indirect to
302 // direct, or direct to different direct.
303 CallGraphNode *CalleeNode;
304 if (Function *Callee = CS.getCalledFunction()) {
305 CalleeNode = CG.getOrInsertFunction(Callee);
306 // Keep track of whether we turned an indirect call into a direct
308 if (!ExistingNode->getFunction()) {
309 DevirtualizedCall = true;
310 DEBUG(dbgs() << " CGSCCPASSMGR: Devirtualized call to '"
311 << Callee->getName() << "'\n");
314 CalleeNode = CG.getCallsExternalNode();
317 // Update the edge target in CGN.
318 CGN->replaceCallEdge(CS, CS, CalleeNode);
323 assert(!CheckingMode &&
324 "CallGraphSCCPass did not update the CallGraph correctly!");
326 // If the call site didn't exist in the CGN yet, add it.
327 CallGraphNode *CalleeNode;
328 if (Function *Callee = CS.getCalledFunction()) {
329 CalleeNode = CG.getOrInsertFunction(Callee);
332 CalleeNode = CG.getCallsExternalNode();
336 CGN->addCalledFunction(CS, CalleeNode);
340 // We scanned the old callgraph node, removing invalidated call sites and
341 // then added back newly found call sites. One thing that can happen is
342 // that an old indirect call site was deleted and replaced with a new direct
343 // call. In this case, we have devirtualized a call, and CGSCCPM would like
344 // to iteratively optimize the new code. Unfortunately, we don't really
345 // have a great way to detect when this happens. As an approximation, we
346 // just look at whether the number of indirect calls is reduced and the
347 // number of direct calls is increased. There are tons of ways to fool this
348 // (e.g. DCE'ing an indirect call and duplicating an unrelated block with a
349 // direct call) but this is close enough.
350 if (NumIndirectRemoved > NumIndirectAdded &&
351 NumDirectRemoved < NumDirectAdded)
352 DevirtualizedCall = true;
354 // After scanning this function, if we still have entries in callsites, then
355 // they are dangling pointers. WeakTrackingVH should save us for this, so
358 assert(CallSites.empty() && "Dangling pointers found in call sites map");
360 // Periodically do an explicit clear to remove tombstones when processing
362 if ((FunctionNo & 15) == 15)
366 DEBUG(if (MadeChange) {
367 dbgs() << "CGSCCPASSMGR: Refreshed SCC is now:\n";
368 for (CallGraphNode *CGN : CurSCC)
370 if (DevirtualizedCall)
371 dbgs() << "CGSCCPASSMGR: Refresh devirtualized a call!\n";
374 dbgs() << "CGSCCPASSMGR: SCC Refresh didn't change call graph.\n";
379 return DevirtualizedCall;
382 /// Execute the body of the entire pass manager on the specified SCC.
383 /// This keeps track of whether a function pass devirtualizes
384 /// any calls and returns it in DevirtualizedCall.
385 bool CGPassManager::RunAllPassesOnSCC(CallGraphSCC &CurSCC, CallGraph &CG,
386 bool &DevirtualizedCall) {
387 bool Changed = false;
389 // Keep track of whether the callgraph is known to be up-to-date or not.
390 // The CGSSC pass manager runs two types of passes:
391 // CallGraphSCC Passes and other random function passes. Because other
392 // random function passes are not CallGraph aware, they may clobber the
393 // call graph by introducing new calls or deleting other ones. This flag
394 // is set to false when we run a function pass so that we know to clean up
395 // the callgraph when we need to run a CGSCCPass again.
396 bool CallGraphUpToDate = true;
398 // Run all passes on current SCC.
399 for (unsigned PassNo = 0, e = getNumContainedPasses();
400 PassNo != e; ++PassNo) {
401 Pass *P = getContainedPass(PassNo);
403 // If we're in -debug-pass=Executions mode, construct the SCC node list,
404 // otherwise avoid constructing this string as it is expensive.
405 if (isPassDebuggingExecutionsOrMore()) {
406 std::string Functions;
408 raw_string_ostream OS(Functions);
409 for (CallGraphSCC::iterator I = CurSCC.begin(), E = CurSCC.end();
411 if (I != CurSCC.begin()) OS << ", ";
416 dumpPassInfo(P, EXECUTION_MSG, ON_CG_MSG, Functions);
420 initializeAnalysisImpl(P);
422 // Actually run this pass on the current SCC.
423 Changed |= RunPassOnSCC(P, CurSCC, CG,
424 CallGraphUpToDate, DevirtualizedCall);
427 dumpPassInfo(P, MODIFICATION_MSG, ON_CG_MSG, "");
430 verifyPreservedAnalysis(P);
431 removeNotPreservedAnalysis(P);
432 recordAvailableAnalysis(P);
433 removeDeadPasses(P, "", ON_CG_MSG);
436 // If the callgraph was left out of date (because the last pass run was a
437 // functionpass), refresh it before we move on to the next SCC.
438 if (!CallGraphUpToDate)
439 DevirtualizedCall |= RefreshCallGraph(CurSCC, CG, false);
443 /// Execute all of the passes scheduled for execution. Keep track of
444 /// whether any of the passes modifies the module, and if so, return true.
445 bool CGPassManager::runOnModule(Module &M) {
446 CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
447 bool Changed = doInitialization(CG);
449 // Walk the callgraph in bottom-up SCC order.
450 scc_iterator<CallGraph*> CGI = scc_begin(&CG);
452 CallGraphSCC CurSCC(CG, &CGI);
453 while (!CGI.isAtEnd()) {
454 // Copy the current SCC and increment past it so that the pass can hack
455 // on the SCC if it wants to without invalidating our iterator.
456 const std::vector<CallGraphNode *> &NodeVec = *CGI;
457 CurSCC.initialize(NodeVec);
460 // At the top level, we run all the passes in this pass manager on the
461 // functions in this SCC. However, we support iterative compilation in the
462 // case where a function pass devirtualizes a call to a function. For
463 // example, it is very common for a function pass (often GVN or instcombine)
464 // to eliminate the addressing that feeds into a call. With that improved
465 // information, we would like the call to be an inline candidate, infer
466 // mod-ref information etc.
468 // Because of this, we allow iteration up to a specified iteration count.
469 // This only happens in the case of a devirtualized call, so we only burn
470 // compile time in the case that we're making progress. We also have a hard
471 // iteration count limit in case there is crazy code.
472 unsigned Iteration = 0;
473 bool DevirtualizedCall = false;
476 dbgs() << " SCCPASSMGR: Re-visiting SCC, iteration #"
477 << Iteration << '\n');
478 DevirtualizedCall = false;
479 Changed |= RunAllPassesOnSCC(CurSCC, CG, DevirtualizedCall);
480 } while (Iteration++ < MaxIterations && DevirtualizedCall);
482 if (DevirtualizedCall)
483 DEBUG(dbgs() << " CGSCCPASSMGR: Stopped iteration after " << Iteration
484 << " times, due to -max-cg-scc-iterations\n");
486 MaxSCCIterations.updateMax(Iteration);
488 Changed |= doFinalization(CG);
493 bool CGPassManager::doInitialization(CallGraph &CG) {
494 bool Changed = false;
495 for (unsigned i = 0, e = getNumContainedPasses(); i != e; ++i) {
496 if (PMDataManager *PM = getContainedPass(i)->getAsPMDataManager()) {
497 assert(PM->getPassManagerType() == PMT_FunctionPassManager &&
498 "Invalid CGPassManager member");
499 Changed |= ((FPPassManager*)PM)->doInitialization(CG.getModule());
501 Changed |= ((CallGraphSCCPass*)getContainedPass(i))->doInitialization(CG);
508 bool CGPassManager::doFinalization(CallGraph &CG) {
509 bool Changed = false;
510 for (unsigned i = 0, e = getNumContainedPasses(); i != e; ++i) {
511 if (PMDataManager *PM = getContainedPass(i)->getAsPMDataManager()) {
512 assert(PM->getPassManagerType() == PMT_FunctionPassManager &&
513 "Invalid CGPassManager member");
514 Changed |= ((FPPassManager*)PM)->doFinalization(CG.getModule());
516 Changed |= ((CallGraphSCCPass*)getContainedPass(i))->doFinalization(CG);
522 //===----------------------------------------------------------------------===//
523 // CallGraphSCC Implementation
524 //===----------------------------------------------------------------------===//
526 /// This informs the SCC and the pass manager that the specified
527 /// Old node has been deleted, and New is to be used in its place.
528 void CallGraphSCC::ReplaceNode(CallGraphNode *Old, CallGraphNode *New) {
529 assert(Old != New && "Should not replace node with self");
530 for (unsigned i = 0; ; ++i) {
531 assert(i != Nodes.size() && "Node not in SCC");
532 if (Nodes[i] != Old) continue;
537 // Update the active scc_iterator so that it doesn't contain dangling
538 // pointers to the old CallGraphNode.
539 scc_iterator<CallGraph*> *CGI = (scc_iterator<CallGraph*>*)Context;
540 CGI->ReplaceNode(Old, New);
543 //===----------------------------------------------------------------------===//
544 // CallGraphSCCPass Implementation
545 //===----------------------------------------------------------------------===//
547 /// Assign pass manager to manage this pass.
548 void CallGraphSCCPass::assignPassManager(PMStack &PMS,
549 PassManagerType PreferredType) {
550 // Find CGPassManager
551 while (!PMS.empty() &&
552 PMS.top()->getPassManagerType() > PMT_CallGraphPassManager)
555 assert(!PMS.empty() && "Unable to handle Call Graph Pass");
558 if (PMS.top()->getPassManagerType() == PMT_CallGraphPassManager)
559 CGP = (CGPassManager*)PMS.top();
561 // Create new Call Graph SCC Pass Manager if it does not exist.
562 assert(!PMS.empty() && "Unable to create Call Graph Pass Manager");
563 PMDataManager *PMD = PMS.top();
565 // [1] Create new Call Graph Pass Manager
566 CGP = new CGPassManager();
568 // [2] Set up new manager's top level manager
569 PMTopLevelManager *TPM = PMD->getTopLevelManager();
570 TPM->addIndirectPassManager(CGP);
572 // [3] Assign manager to manage this new manager. This may create
573 // and push new managers into PMS
575 TPM->schedulePass(P);
577 // [4] Push new manager into PMS
584 /// For this class, we declare that we require and preserve the call graph.
585 /// If the derived class implements this method, it should
586 /// always explicitly call the implementation here.
587 void CallGraphSCCPass::getAnalysisUsage(AnalysisUsage &AU) const {
588 AU.addRequired<CallGraphWrapperPass>();
589 AU.addPreserved<CallGraphWrapperPass>();
592 //===----------------------------------------------------------------------===//
593 // PrintCallGraphPass Implementation
594 //===----------------------------------------------------------------------===//
598 /// PrintCallGraphPass - Print a Module corresponding to a call graph.
600 class PrintCallGraphPass : public CallGraphSCCPass {
602 raw_ostream &OS; // raw_ostream to print on.
607 PrintCallGraphPass(const std::string &B, raw_ostream &OS)
608 : CallGraphSCCPass(ID), Banner(B), OS(OS) {}
610 void getAnalysisUsage(AnalysisUsage &AU) const override {
611 AU.setPreservesAll();
614 bool runOnSCC(CallGraphSCC &SCC) override {
615 bool BannerPrinted = false;
616 auto PrintBannerOnce = [&] () {
620 BannerPrinted = true;
622 for (CallGraphNode *CGN : SCC) {
623 if (Function *F = CGN->getFunction()) {
624 if (!F->isDeclaration() && isFunctionInPrintList(F->getName())) {
628 } else if (isFunctionInPrintList("*")) {
630 OS << "\nPrinting <null> Function\n";
636 StringRef getPassName() const override { return "Print CallGraph IR"; }
639 } // end anonymous namespace.
641 char PrintCallGraphPass::ID = 0;
643 Pass *CallGraphSCCPass::createPrinterPass(raw_ostream &OS,
644 const std::string &Banner) const {
645 return new PrintCallGraphPass(Banner, OS);
648 bool CallGraphSCCPass::skipSCC(CallGraphSCC &SCC) const {
649 return !SCC.getCallGraph().getModule()
652 .shouldRunPass(this, SCC);
655 char DummyCGSCCPass::ID = 0;
657 INITIALIZE_PASS(DummyCGSCCPass, "DummyCGSCCPass", "DummyCGSCCPass", false,