1 //===- ShrinkWrap.cpp - Compute safe point for prolog/epilog insertion ----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass looks for safe point where the prologue and epilogue can be
12 // The safe point for the prologue (resp. epilogue) is called Save
14 // A point is safe for prologue (resp. epilogue) if and only if
15 // it 1) dominates (resp. post-dominates) all the frame related operations and
16 // between 2) two executions of the Save (resp. Restore) point there is an
17 // execution of the Restore (resp. Save) point.
19 // For instance, the following points are safe:
20 // for (int i = 0; i < 10; ++i) {
25 // Indeed, the execution looks like Save -> Restore -> Save -> Restore ...
26 // And the following points are not:
27 // for (int i = 0; i < 10; ++i) {
31 // for (int i = 0; i < 10; ++i) {
35 // Indeed, the execution looks like Save -> Save -> ... -> Restore -> Restore.
37 // This pass also ensures that the safe points are 3) cheaper than the regular
38 // entry and exits blocks.
40 // Property #1 is ensured via the use of MachineDominatorTree and
41 // MachinePostDominatorTree.
42 // Property #2 is ensured via property #1 and MachineLoopInfo, i.e., both
43 // points must be in the same loop.
44 // Property #3 is ensured via the MachineBlockFrequencyInfo.
46 // If this pass found points matching all these properties, then
47 // MachineFrameInfo is updated with this information.
49 //===----------------------------------------------------------------------===//
51 #include "llvm/ADT/BitVector.h"
52 #include "llvm/ADT/PostOrderIterator.h"
53 #include "llvm/ADT/SetVector.h"
54 #include "llvm/ADT/SmallVector.h"
55 #include "llvm/ADT/Statistic.h"
56 #include "llvm/CodeGen/MachineBasicBlock.h"
57 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
58 #include "llvm/CodeGen/MachineDominators.h"
59 #include "llvm/CodeGen/MachineFrameInfo.h"
60 #include "llvm/CodeGen/MachineFunction.h"
61 #include "llvm/CodeGen/MachineFunctionPass.h"
62 #include "llvm/CodeGen/MachineInstr.h"
63 #include "llvm/CodeGen/MachineLoopInfo.h"
64 #include "llvm/CodeGen/MachineOperand.h"
65 #include "llvm/CodeGen/MachinePostDominators.h"
66 #include "llvm/CodeGen/RegisterClassInfo.h"
67 #include "llvm/CodeGen/RegisterScavenging.h"
68 #include "llvm/CodeGen/TargetFrameLowering.h"
69 #include "llvm/CodeGen/TargetInstrInfo.h"
70 #include "llvm/CodeGen/TargetRegisterInfo.h"
71 #include "llvm/CodeGen/TargetSubtargetInfo.h"
72 #include "llvm/IR/Attributes.h"
73 #include "llvm/IR/Function.h"
74 #include "llvm/MC/MCAsmInfo.h"
75 #include "llvm/Pass.h"
76 #include "llvm/Support/CommandLine.h"
77 #include "llvm/Support/Debug.h"
78 #include "llvm/Support/ErrorHandling.h"
79 #include "llvm/Support/raw_ostream.h"
80 #include "llvm/Target/TargetMachine.h"
87 #define DEBUG_TYPE "shrink-wrap"
89 STATISTIC(NumFunc, "Number of functions");
90 STATISTIC(NumCandidates, "Number of shrink-wrapping candidates");
91 STATISTIC(NumCandidatesDropped,
92 "Number of shrink-wrapping candidates dropped because of frequency");
94 static cl::opt<cl::boolOrDefault>
95 EnableShrinkWrapOpt("enable-shrink-wrap", cl::Hidden,
96 cl::desc("enable the shrink-wrapping pass"));
100 /// \brief Class to determine where the safe point to insert the
101 /// prologue and epilogue are.
102 /// Unlike the paper from Fred C. Chow, PLDI'88, that introduces the
103 /// shrink-wrapping term for prologue/epilogue placement, this pass
104 /// does not rely on expensive data-flow analysis. Instead we use the
105 /// dominance properties and loop information to decide which point
106 /// are safe for such insertion.
107 class ShrinkWrap : public MachineFunctionPass {
108 /// Hold callee-saved information.
109 RegisterClassInfo RCI;
110 MachineDominatorTree *MDT;
111 MachinePostDominatorTree *MPDT;
113 /// Current safe point found for the prologue.
114 /// The prologue will be inserted before the first instruction
115 /// in this basic block.
116 MachineBasicBlock *Save;
118 /// Current safe point found for the epilogue.
119 /// The epilogue will be inserted before the first terminator instruction
120 /// in this basic block.
121 MachineBasicBlock *Restore;
123 /// Hold the information of the basic block frequency.
124 /// Use to check the profitability of the new points.
125 MachineBlockFrequencyInfo *MBFI;
127 /// Hold the loop information. Used to determine if Save and Restore
128 /// are in the same loop.
129 MachineLoopInfo *MLI;
131 /// Frequency of the Entry block.
134 /// Current opcode for frame setup.
135 unsigned FrameSetupOpcode;
137 /// Current opcode for frame destroy.
138 unsigned FrameDestroyOpcode;
141 const MachineBasicBlock *Entry;
143 using SetOfRegs = SmallSetVector<unsigned, 16>;
145 /// Registers that need to be saved for the current function.
146 mutable SetOfRegs CurrentCSRs;
148 /// Current MachineFunction.
149 MachineFunction *MachineFunc;
151 /// \brief Check if \p MI uses or defines a callee-saved register or
152 /// a frame index. If this is the case, this means \p MI must happen
153 /// after Save and before Restore.
154 bool useOrDefCSROrFI(const MachineInstr &MI, RegScavenger *RS) const;
156 const SetOfRegs &getCurrentCSRs(RegScavenger *RS) const {
157 if (CurrentCSRs.empty()) {
159 const TargetFrameLowering *TFI =
160 MachineFunc->getSubtarget().getFrameLowering();
162 TFI->determineCalleeSaves(*MachineFunc, SavedRegs, RS);
164 for (int Reg = SavedRegs.find_first(); Reg != -1;
165 Reg = SavedRegs.find_next(Reg))
166 CurrentCSRs.insert((unsigned)Reg);
171 /// \brief Update the Save and Restore points such that \p MBB is in
172 /// the region that is dominated by Save and post-dominated by Restore
173 /// and Save and Restore still match the safe point definition.
174 /// Such point may not exist and Save and/or Restore may be null after
176 void updateSaveRestorePoints(MachineBasicBlock &MBB, RegScavenger *RS);
178 /// \brief Initialize the pass for \p MF.
179 void init(MachineFunction &MF) {
180 RCI.runOnMachineFunction(MF);
181 MDT = &getAnalysis<MachineDominatorTree>();
182 MPDT = &getAnalysis<MachinePostDominatorTree>();
185 MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
186 MLI = &getAnalysis<MachineLoopInfo>();
187 EntryFreq = MBFI->getEntryFreq();
188 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
189 FrameSetupOpcode = TII.getCallFrameSetupOpcode();
190 FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
198 /// Check whether or not Save and Restore points are still interesting for
200 bool ArePointsInteresting() const { return Save != Entry && Save && Restore; }
202 /// \brief Check if shrink wrapping is enabled for this target and function.
203 static bool isShrinkWrapEnabled(const MachineFunction &MF);
208 ShrinkWrap() : MachineFunctionPass(ID) {
209 initializeShrinkWrapPass(*PassRegistry::getPassRegistry());
212 void getAnalysisUsage(AnalysisUsage &AU) const override {
213 AU.setPreservesAll();
214 AU.addRequired<MachineBlockFrequencyInfo>();
215 AU.addRequired<MachineDominatorTree>();
216 AU.addRequired<MachinePostDominatorTree>();
217 AU.addRequired<MachineLoopInfo>();
218 MachineFunctionPass::getAnalysisUsage(AU);
221 StringRef getPassName() const override { return "Shrink Wrapping analysis"; }
223 /// \brief Perform the shrink-wrapping analysis and update
224 /// the MachineFrameInfo attached to \p MF with the results.
225 bool runOnMachineFunction(MachineFunction &MF) override;
228 } // end anonymous namespace
230 char ShrinkWrap::ID = 0;
232 char &llvm::ShrinkWrapID = ShrinkWrap::ID;
234 INITIALIZE_PASS_BEGIN(ShrinkWrap, DEBUG_TYPE, "Shrink Wrap Pass", false, false)
235 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
236 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
237 INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
238 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
239 INITIALIZE_PASS_END(ShrinkWrap, DEBUG_TYPE, "Shrink Wrap Pass", false, false)
241 bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr &MI,
242 RegScavenger *RS) const {
243 // Ignore DBG_VALUE and other meta instructions that must not affect codegen.
244 if (MI.isMetaInstruction())
247 if (MI.getOpcode() == FrameSetupOpcode ||
248 MI.getOpcode() == FrameDestroyOpcode) {
249 DEBUG(dbgs() << "Frame instruction: " << MI << '\n');
252 for (const MachineOperand &MO : MI.operands()) {
253 bool UseOrDefCSR = false;
255 unsigned PhysReg = MO.getReg();
258 assert(TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
259 "Unallocated register?!");
260 UseOrDefCSR = RCI.getLastCalleeSavedAlias(PhysReg);
261 } else if (MO.isRegMask()) {
262 // Check if this regmask clobbers any of the CSRs.
263 for (unsigned Reg : getCurrentCSRs(RS)) {
264 if (MO.clobbersPhysReg(Reg)) {
270 if (UseOrDefCSR || MO.isFI()) {
271 DEBUG(dbgs() << "Use or define CSR(" << UseOrDefCSR << ") or FI("
272 << MO.isFI() << "): " << MI << '\n');
279 /// \brief Helper function to find the immediate (post) dominator.
280 template <typename ListOfBBs, typename DominanceAnalysis>
281 static MachineBasicBlock *FindIDom(MachineBasicBlock &Block, ListOfBBs BBs,
282 DominanceAnalysis &Dom) {
283 MachineBasicBlock *IDom = &Block;
284 for (MachineBasicBlock *BB : BBs) {
285 IDom = Dom.findNearestCommonDominator(IDom, BB);
294 void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock &MBB,
296 // Get rid of the easy cases first.
300 Save = MDT->findNearestCommonDominator(Save, &MBB);
303 DEBUG(dbgs() << "Found a block that is not reachable from Entry\n");
309 else if (MPDT->getNode(&MBB)) // If the block is not in the post dom tree, it
310 // means the block never returns. If that's the
311 // case, we don't want to call
312 // `findNearestCommonDominator`, which will
314 Restore = MPDT->findNearestCommonDominator(Restore, &MBB);
316 Restore = nullptr; // Abort, we can't find a restore point in this case.
318 // Make sure we would be able to insert the restore code before the
320 if (Restore == &MBB) {
321 for (const MachineInstr &Terminator : MBB.terminators()) {
322 if (!useOrDefCSROrFI(Terminator, RS))
324 // One of the terminator needs to happen before the restore point.
325 if (MBB.succ_empty()) {
326 Restore = nullptr; // Abort, we can't find a restore point in this case.
329 // Look for a restore point that post-dominates all the successors.
330 // The immediate post-dominator is what we are looking for.
331 Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
337 DEBUG(dbgs() << "Restore point needs to be spanned on several blocks\n");
341 // Make sure Save and Restore are suitable for shrink-wrapping:
342 // 1. all path from Save needs to lead to Restore before exiting.
343 // 2. all path to Restore needs to go through Save from Entry.
344 // We achieve that by making sure that:
345 // A. Save dominates Restore.
346 // B. Restore post-dominates Save.
347 // C. Save and Restore are in the same loop.
348 bool SaveDominatesRestore = false;
349 bool RestorePostDominatesSave = false;
350 while (Save && Restore &&
351 (!(SaveDominatesRestore = MDT->dominates(Save, Restore)) ||
352 !(RestorePostDominatesSave = MPDT->dominates(Restore, Save)) ||
353 // Post-dominance is not enough in loops to ensure that all uses/defs
354 // are after the prologue and before the epilogue at runtime.
363 // All the uses/defs of CSRs are dominated by Save and post-dominated
364 // by Restore. However, the CSRs uses are still reachable after
365 // Restore and before Save are executed.
367 // For now, just push the restore/save points outside of loops.
368 // FIXME: Refine the criteria to still find interesting cases
370 MLI->getLoopFor(Save) || MLI->getLoopFor(Restore))) {
372 if (!SaveDominatesRestore) {
373 Save = MDT->findNearestCommonDominator(Save, Restore);
377 if (!RestorePostDominatesSave)
378 Restore = MPDT->findNearestCommonDominator(Restore, Save);
381 if (Save && Restore &&
382 (MLI->getLoopFor(Save) || MLI->getLoopFor(Restore))) {
383 if (MLI->getLoopDepth(Save) > MLI->getLoopDepth(Restore)) {
384 // Push Save outside of this loop if immediate dominator is different
385 // from save block. If immediate dominator is not different, bail out.
386 Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
390 // If the loop does not exit, there is no point in looking
391 // for a post-dominator outside the loop.
392 SmallVector<MachineBasicBlock*, 4> ExitBlocks;
393 MLI->getLoopFor(Restore)->getExitingBlocks(ExitBlocks);
394 // Push Restore outside of this loop.
395 // Look for the immediate post-dominator of the loop exits.
396 MachineBasicBlock *IPdom = Restore;
397 for (MachineBasicBlock *LoopExitBB: ExitBlocks) {
398 IPdom = FindIDom<>(*IPdom, LoopExitBB->successors(), *MPDT);
402 // If the immediate post-dominator is not in a less nested loop,
403 // then we are stuck in a program with an infinite loop.
404 // In that case, we will not find a safe point, hence, bail out.
405 if (IPdom && MLI->getLoopDepth(IPdom) < MLI->getLoopDepth(Restore))
416 /// Check whether the edge (\p SrcBB, \p DestBB) is a backedge according to MLI.
417 /// I.e., check if it exists a loop that contains SrcBB and where DestBB is the
419 static bool isProperBackedge(const MachineLoopInfo &MLI,
420 const MachineBasicBlock *SrcBB,
421 const MachineBasicBlock *DestBB) {
422 for (const MachineLoop *Loop = MLI.getLoopFor(SrcBB); Loop;
423 Loop = Loop->getParentLoop()) {
424 if (Loop->getHeader() == DestBB)
430 /// Check if the CFG of \p MF is irreducible.
431 static bool isIrreducibleCFG(const MachineFunction &MF,
432 const MachineLoopInfo &MLI) {
433 const MachineBasicBlock *Entry = &*MF.begin();
434 ReversePostOrderTraversal<const MachineBasicBlock *> RPOT(Entry);
435 BitVector VisitedBB(MF.getNumBlockIDs());
436 for (const MachineBasicBlock *MBB : RPOT) {
437 VisitedBB.set(MBB->getNumber());
438 for (const MachineBasicBlock *SuccBB : MBB->successors()) {
439 if (!VisitedBB.test(SuccBB->getNumber()))
441 // We already visited SuccBB, thus MBB->SuccBB must be a backedge.
442 // Check that the head matches what we have in the loop information.
443 // Otherwise, we have an irreducible graph.
444 if (!isProperBackedge(MLI, MBB, SuccBB))
451 bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) {
452 if (skipFunction(MF.getFunction()) || MF.empty() || !isShrinkWrapEnabled(MF))
455 DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n');
459 if (isIrreducibleCFG(MF, *MLI)) {
460 // If MF is irreducible, a block may be in a loop without
461 // MachineLoopInfo reporting it. I.e., we may use the
462 // post-dominance property in loops, which lead to incorrect
463 // results. Moreover, we may miss that the prologue and
464 // epilogue are not in the same loop, leading to unbalanced
465 // construction/deconstruction of the stack frame.
466 DEBUG(dbgs() << "Irreducible CFGs are not supported yet\n");
470 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
471 std::unique_ptr<RegScavenger> RS(
472 TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr);
474 for (MachineBasicBlock &MBB : MF) {
475 DEBUG(dbgs() << "Look into: " << MBB.getNumber() << ' ' << MBB.getName()
478 if (MBB.isEHFuncletEntry()) {
479 DEBUG(dbgs() << "EH Funclets are not supported yet.\n");
483 for (const MachineInstr &MI : MBB) {
484 if (!useOrDefCSROrFI(MI, RS.get()))
486 // Save (resp. restore) point must dominate (resp. post dominate)
487 // MI. Look for the proper basic block for those.
488 updateSaveRestorePoints(MBB, RS.get());
489 // If we are at a point where we cannot improve the placement of
490 // save/restore instructions, just give up.
491 if (!ArePointsInteresting()) {
492 DEBUG(dbgs() << "No Shrink wrap candidate found\n");
495 // No need to look for other instructions, this basic block
496 // will already be part of the handled region.
500 if (!ArePointsInteresting()) {
501 // If the points are not interesting at this point, then they must be null
502 // because it means we did not encounter any frame/CSR related code.
503 // Otherwise, we would have returned from the previous loop.
504 assert(!Save && !Restore && "We miss a shrink-wrap opportunity?!");
505 DEBUG(dbgs() << "Nothing to shrink-wrap\n");
509 DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq
512 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
514 DEBUG(dbgs() << "Shrink wrap candidates (#, Name, Freq):\nSave: "
515 << Save->getNumber() << ' ' << Save->getName() << ' '
516 << MBFI->getBlockFreq(Save).getFrequency() << "\nRestore: "
517 << Restore->getNumber() << ' ' << Restore->getName() << ' '
518 << MBFI->getBlockFreq(Restore).getFrequency() << '\n');
520 bool IsSaveCheap, TargetCanUseSaveAsPrologue = false;
521 if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save).getFrequency()) &&
522 EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) &&
523 ((TargetCanUseSaveAsPrologue = TFI->canUseAsPrologue(*Save)) &&
524 TFI->canUseAsEpilogue(*Restore)))
526 DEBUG(dbgs() << "New points are too expensive or invalid for the target\n");
527 MachineBasicBlock *NewBB;
528 if (!IsSaveCheap || !TargetCanUseSaveAsPrologue) {
529 Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
534 // Restore is expensive.
535 Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
540 updateSaveRestorePoints(*NewBB, RS.get());
541 } while (Save && Restore);
543 if (!ArePointsInteresting()) {
544 ++NumCandidatesDropped;
548 DEBUG(dbgs() << "Final shrink wrap candidates:\nSave: " << Save->getNumber()
549 << ' ' << Save->getName() << "\nRestore: "
550 << Restore->getNumber() << ' ' << Restore->getName() << '\n');
552 MachineFrameInfo &MFI = MF.getFrameInfo();
553 MFI.setSavePoint(Save);
554 MFI.setRestorePoint(Restore);
559 bool ShrinkWrap::isShrinkWrapEnabled(const MachineFunction &MF) {
560 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
562 switch (EnableShrinkWrapOpt) {
564 return TFI->enableShrinkWrapping(MF) &&
565 // Windows with CFI has some limitations that make it impossible
566 // to use shrink-wrapping.
567 !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
568 // Sanitizers look at the value of the stack at the location
569 // of the crash. Since a crash can happen anywhere, the
570 // frame must be lowered before anything else happen for the
571 // sanitizers to be able to get a correct stack frame.
572 !(MF.getFunction().hasFnAttribute(Attribute::SanitizeAddress) ||
573 MF.getFunction().hasFnAttribute(Attribute::SanitizeThread) ||
574 MF.getFunction().hasFnAttribute(Attribute::SanitizeMemory) ||
575 MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress));
576 // If EnableShrinkWrap is set, it takes precedence on whatever the
577 // target sets. The rational is that we assume we want to test
578 // something related to shrink-wrapping.
584 llvm_unreachable("Invalid shrink-wrapping state");