1 //===-- StackColoring.cpp -------------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass implements the stack-coloring optimization that looks for
11 // lifetime markers machine instructions (LIFESTART_BEGIN and LIFESTART_END),
12 // which represent the possible lifetime of stack slots. It attempts to
13 // merge disjoint stack slots and reduce the used stack space.
14 // NOTE: This pass is not StackSlotColoring, which optimizes spill slots.
16 // TODO: In the future we plan to improve stack coloring in the following ways:
17 // 1. Allow merging multiple small slots into a single larger slot at different
19 // 2. Merge this pass with StackSlotColoring and allow merging of allocas with
22 //===----------------------------------------------------------------------===//
24 #include "llvm/ADT/BitVector.h"
25 #include "llvm/ADT/DepthFirstIterator.h"
26 #include "llvm/ADT/SetVector.h"
27 #include "llvm/ADT/SmallPtrSet.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/Analysis/ValueTracking.h"
30 #include "llvm/CodeGen/LiveInterval.h"
31 #include "llvm/CodeGen/MachineBasicBlock.h"
32 #include "llvm/CodeGen/MachineFrameInfo.h"
33 #include "llvm/CodeGen/MachineFunctionPass.h"
34 #include "llvm/CodeGen/MachineLoopInfo.h"
35 #include "llvm/CodeGen/MachineMemOperand.h"
36 #include "llvm/CodeGen/MachineModuleInfo.h"
37 #include "llvm/CodeGen/MachineRegisterInfo.h"
38 #include "llvm/CodeGen/Passes.h"
39 #include "llvm/CodeGen/PseudoSourceValue.h"
40 #include "llvm/CodeGen/SlotIndexes.h"
41 #include "llvm/CodeGen/StackProtector.h"
42 #include "llvm/CodeGen/WinEHFuncInfo.h"
43 #include "llvm/IR/DebugInfo.h"
44 #include "llvm/IR/Function.h"
45 #include "llvm/IR/Instructions.h"
46 #include "llvm/IR/IntrinsicInst.h"
47 #include "llvm/IR/Module.h"
48 #include "llvm/Support/CommandLine.h"
49 #include "llvm/Support/Debug.h"
50 #include "llvm/Support/raw_ostream.h"
51 #include "llvm/Target/TargetInstrInfo.h"
52 #include "llvm/Target/TargetRegisterInfo.h"
56 #define DEBUG_TYPE "stack-coloring"
59 DisableColoring("no-stack-coloring",
60 cl::init(false), cl::Hidden,
61 cl::desc("Disable stack coloring"));
63 /// The user may write code that uses allocas outside of the declared lifetime
64 /// zone. This can happen when the user returns a reference to a local
65 /// data-structure. We can detect these cases and decide not to optimize the
66 /// code. If this flag is enabled, we try to save the user. This option
67 /// is treated as overriding LifetimeStartOnFirstUse below.
69 ProtectFromEscapedAllocas("protect-from-escaped-allocas",
70 cl::init(false), cl::Hidden,
71 cl::desc("Do not optimize lifetime zones that "
74 /// Enable enhanced dataflow scheme for lifetime analysis (treat first
75 /// use of stack slot as start of slot lifetime, as opposed to looking
76 /// for LIFETIME_START marker). See "Implementation notes" below for
79 LifetimeStartOnFirstUse("stackcoloring-lifetime-start-on-first-use",
80 cl::init(true), cl::Hidden,
81 cl::desc("Treat stack lifetimes as starting on first use, not on START marker."));
84 STATISTIC(NumMarkerSeen, "Number of lifetime markers found.");
85 STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
86 STATISTIC(StackSlotMerged, "Number of stack slot merged.");
87 STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
90 // Implementation Notes:
91 // ---------------------
93 // Consider the following motivating example:
96 // char b1[1024], b2[1024];
102 // char b4[1024], b5[1024];
103 // <uses of b2, b4, b5>;
108 // In the code above, "b3" and "b4" are declared in distinct lexical
109 // scopes, meaning that it is easy to prove that they can share the
110 // same stack slot. Variables "b1" and "b2" are declared in the same
111 // scope, meaning that from a lexical point of view, their lifetimes
112 // overlap. From a control flow pointer of view, however, the two
113 // variables are accessed in disjoint regions of the CFG, thus it
114 // should be possible for them to share the same stack slot. An ideal
115 // stack allocation for the function above would look like:
121 // Achieving this allocation is tricky, however, due to the way
122 // lifetime markers are inserted. Here is a simplified view of the
123 // control flow graph for the code above:
125 // +------ block 0 -------+
126 // 0| LIFETIME_START b1, b2 |
127 // 1| <test 'if' condition> |
128 // +-----------------------+
130 // +------ block 1 -------+ +------ block 2 -------+
131 // 2| LIFETIME_START b3 | 5| LIFETIME_START b4, b5 |
132 // 3| <uses of b1, b3> | 6| <uses of b2, b4, b5> |
133 // 4| LIFETIME_END b3 | 7| LIFETIME_END b4, b5 |
134 // +-----------------------+ +-----------------------+
136 // +------ block 3 -------+
137 // 8| <cleanupcode> |
138 // 9| LIFETIME_END b1, b2 |
140 // +-----------------------+
142 // If we create live intervals for the variables above strictly based
143 // on the lifetime markers, we'll get the set of intervals on the
144 // left. If we ignore the lifetime start markers and instead treat a
145 // variable's lifetime as beginning with the first reference to the
146 // var, then we get the intervals on the right.
148 // LIFETIME_START First Use
149 // b1: [0,9] [3,4] [8,9]
155 // For the intervals on the left, the best we can do is overlap two
156 // variables (b3 and b4, for example); this gives us a stack size of
157 // 4*1024 bytes, not ideal. When treating first-use as the start of a
158 // lifetime, we can additionally overlap b1 and b5, giving us a 3*1024
159 // byte stack (better).
161 // Relying entirely on first-use of stack slots is problematic,
162 // however, due to the fact that optimizations can sometimes migrate
163 // uses of a variable outside of its lifetime start/end region. Here
167 // char b1[1024], b2[1024];
180 // Before optimization, the control flow graph for the code above
181 // might look like the following:
183 // +------ block 0 -------+
184 // 0| LIFETIME_START b1, b2 |
185 // 1| <test 'if' condition> |
186 // +-----------------------+
188 // +------ block 1 -------+ +------- block 2 -------+
189 // 2| <uses of b2> | 3| <uses of b1> |
190 // +-----------------------+ +-----------------------+
192 // | +------- block 3 -------+ <-\.
193 // | 4| <while condition> | |
194 // | +-----------------------+ |
196 // | / +------- block 4 -------+
197 // \ / 5| LIFETIME_START b3 | |
198 // \ / 6| <uses of b3> | |
199 // \ / 7| LIFETIME_END b3 | |
200 // \ | +------------------------+ |
202 // +------ block 5 -----+ \---------------
203 // 8| <cleanupcode> |
204 // 9| LIFETIME_END b1, b2 |
206 // +---------------------+
208 // During optimization, however, it can happen that an instruction
209 // computing an address in "b3" (for example, a loop-invariant GEP) is
210 // hoisted up out of the loop from block 4 to block 2. [Note that
211 // this is not an actual load from the stack, only an instruction that
212 // computes the address to be loaded]. If this happens, there is now a
213 // path leading from the first use of b3 to the return instruction
214 // that does not encounter the b3 LIFETIME_END, hence b3's lifetime is
215 // now larger than if we were computing live intervals strictly based
216 // on lifetime markers. In the example above, this lengthened lifetime
217 // would mean that it would appear illegal to overlap b3 with b2.
219 // To deal with this such cases, the code in ::collectMarkers() below
220 // tries to identify "degenerate" slots -- those slots where on a single
221 // forward pass through the CFG we encounter a first reference to slot
222 // K before we hit the slot K lifetime start marker. For such slots,
223 // we fall back on using the lifetime start marker as the beginning of
224 // the variable's lifetime. NB: with this implementation, slots can
225 // appear degenerate in cases where there is unstructured control flow:
230 // memcpy(&b[0], ...);
235 // If in RPO ordering chosen to walk the CFG we happen to visit the b[k]
236 // before visiting the memcpy block (which will contain the lifetime start
237 // for "b" then it will appear that 'b' has a degenerate lifetime.
240 //===----------------------------------------------------------------------===//
241 // StackColoring Pass
242 //===----------------------------------------------------------------------===//
245 /// StackColoring - A machine pass for merging disjoint stack allocations,
246 /// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
247 class StackColoring : public MachineFunctionPass {
248 MachineFrameInfo *MFI;
251 /// A class representing liveness information for a single basic block.
252 /// Each bit in the BitVector represents the liveness property
253 /// for a different stack slot.
254 struct BlockLifetimeInfo {
255 /// Which slots BEGINs in each basic block.
257 /// Which slots ENDs in each basic block.
259 /// Which slots are marked as LIVE_IN, coming into each basic block.
261 /// Which slots are marked as LIVE_OUT, coming out of each basic block.
265 /// Maps active slots (per bit) for each basic block.
266 typedef DenseMap<const MachineBasicBlock*, BlockLifetimeInfo> LivenessMap;
267 LivenessMap BlockLiveness;
269 /// Maps serial numbers to basic blocks.
270 DenseMap<const MachineBasicBlock*, int> BasicBlocks;
271 /// Maps basic blocks to a serial number.
272 SmallVector<const MachineBasicBlock*, 8> BasicBlockNumbering;
274 /// Maps liveness intervals for each slot.
275 SmallVector<std::unique_ptr<LiveInterval>, 16> Intervals;
276 /// VNInfo is used for the construction of LiveIntervals.
277 VNInfo::Allocator VNInfoAllocator;
278 /// SlotIndex analysis object.
279 SlotIndexes *Indexes;
280 /// The stack protector object.
283 /// The list of lifetime markers found. These markers are to be removed
284 /// once the coloring is done.
285 SmallVector<MachineInstr*, 8> Markers;
287 /// Record the FI slots for which we have seen some sort of
288 /// lifetime marker (either start or end).
289 BitVector InterestingSlots;
291 /// FI slots that need to be handled conservatively (for these
292 /// slots lifetime-start-on-first-use is disabled).
293 BitVector ConservativeSlots;
295 /// Number of iterations taken during data flow analysis.
296 unsigned NumIterations;
300 StackColoring() : MachineFunctionPass(ID) {
301 initializeStackColoringPass(*PassRegistry::getPassRegistry());
303 void getAnalysisUsage(AnalysisUsage &AU) const override;
304 bool runOnMachineFunction(MachineFunction &MF) override;
309 void dumpIntervals() const;
310 void dumpBB(MachineBasicBlock *MBB) const;
311 void dumpBV(const char *tag, const BitVector &BV) const;
313 /// Removes all of the lifetime marker instructions from the function.
314 /// \returns true if any markers were removed.
315 bool removeAllMarkers();
317 /// Scan the machine function and find all of the lifetime markers.
318 /// Record the findings in the BEGIN and END vectors.
319 /// \returns the number of markers found.
320 unsigned collectMarkers(unsigned NumSlot);
322 /// Perform the dataflow calculation and calculate the lifetime for each of
323 /// the slots, based on the BEGIN/END vectors. Set the LifetimeLIVE_IN and
324 /// LifetimeLIVE_OUT maps that represent which stack slots are live coming
325 /// in and out blocks.
326 void calculateLocalLiveness();
328 /// Returns TRUE if we're using the first-use-begins-lifetime method for
329 /// this slot (if FALSE, then the start marker is treated as start of lifetime).
330 bool applyFirstUse(int Slot) {
331 if (!LifetimeStartOnFirstUse || ProtectFromEscapedAllocas)
333 if (ConservativeSlots.test(Slot))
338 /// Examines the specified instruction and returns TRUE if the instruction
339 /// represents the start or end of an interesting lifetime. The slot or slots
340 /// starting or ending are added to the vector "slots" and "isStart" is set
342 /// \returns True if inst contains a lifetime start or end
343 bool isLifetimeStartOrEnd(const MachineInstr &MI,
344 SmallVector<int, 4> &slots,
347 /// Construct the LiveIntervals for the slots.
348 void calculateLiveIntervals(unsigned NumSlots);
350 /// Go over the machine function and change instructions which use stack
351 /// slots to use the joint slots.
352 void remapInstructions(DenseMap<int, int> &SlotRemap);
354 /// The input program may contain instructions which are not inside lifetime
355 /// markers. This can happen due to a bug in the compiler or due to a bug in
356 /// user code (for example, returning a reference to a local variable).
357 /// This procedure checks all of the instructions in the function and
358 /// invalidates lifetime ranges which do not contain all of the instructions
359 /// which access that frame slot.
360 void removeInvalidSlotRanges();
362 /// Map entries which point to other entries to their destination.
363 /// A->B->C becomes A->C.
364 void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
366 /// Used in collectMarkers
367 typedef DenseMap<const MachineBasicBlock*, BitVector> BlockBitVecMap;
369 } // end anonymous namespace
371 char StackColoring::ID = 0;
372 char &llvm::StackColoringID = StackColoring::ID;
374 INITIALIZE_PASS_BEGIN(StackColoring, DEBUG_TYPE,
375 "Merge disjoint stack slots", false, false)
376 INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
377 INITIALIZE_PASS_DEPENDENCY(StackProtector)
378 INITIALIZE_PASS_END(StackColoring, DEBUG_TYPE,
379 "Merge disjoint stack slots", false, false)
381 void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
382 AU.addRequired<SlotIndexes>();
383 AU.addRequired<StackProtector>();
384 MachineFunctionPass::getAnalysisUsage(AU);
387 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
388 LLVM_DUMP_METHOD void StackColoring::dumpBV(const char *tag,
389 const BitVector &BV) const {
390 dbgs() << tag << " : { ";
391 for (unsigned I = 0, E = BV.size(); I != E; ++I)
392 dbgs() << BV.test(I) << " ";
396 LLVM_DUMP_METHOD void StackColoring::dumpBB(MachineBasicBlock *MBB) const {
397 LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
398 assert(BI != BlockLiveness.end() && "Block not found");
399 const BlockLifetimeInfo &BlockInfo = BI->second;
401 dumpBV("BEGIN", BlockInfo.Begin);
402 dumpBV("END", BlockInfo.End);
403 dumpBV("LIVE_IN", BlockInfo.LiveIn);
404 dumpBV("LIVE_OUT", BlockInfo.LiveOut);
407 LLVM_DUMP_METHOD void StackColoring::dump() const {
408 for (MachineBasicBlock *MBB : depth_first(MF)) {
409 dbgs() << "Inspecting block #" << MBB->getNumber() << " ["
410 << MBB->getName() << "]\n";
415 LLVM_DUMP_METHOD void StackColoring::dumpIntervals() const {
416 for (unsigned I = 0, E = Intervals.size(); I != E; ++I) {
417 dbgs() << "Interval[" << I << "]:\n";
418 Intervals[I]->dump();
423 static inline int getStartOrEndSlot(const MachineInstr &MI)
425 assert((MI.getOpcode() == TargetOpcode::LIFETIME_START ||
426 MI.getOpcode() == TargetOpcode::LIFETIME_END) &&
427 "Expected LIFETIME_START or LIFETIME_END op");
428 const MachineOperand &MO = MI.getOperand(0);
429 int Slot = MO.getIndex();
436 // At the moment the only way to end a variable lifetime is with
437 // a VARIABLE_LIFETIME op (which can't contain a start). If things
438 // change and the IR allows for a single inst that both begins
439 // and ends lifetime(s), this interface will need to be reworked.
441 bool StackColoring::isLifetimeStartOrEnd(const MachineInstr &MI,
442 SmallVector<int, 4> &slots,
445 if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
446 MI.getOpcode() == TargetOpcode::LIFETIME_END) {
447 int Slot = getStartOrEndSlot(MI);
450 if (!InterestingSlots.test(Slot))
452 slots.push_back(Slot);
453 if (MI.getOpcode() == TargetOpcode::LIFETIME_END) {
457 if (! applyFirstUse(Slot)) {
461 } else if (LifetimeStartOnFirstUse && !ProtectFromEscapedAllocas) {
462 if (! MI.isDebugValue()) {
464 for (const MachineOperand &MO : MI.operands()) {
467 int Slot = MO.getIndex();
470 if (InterestingSlots.test(Slot) && applyFirstUse(Slot)) {
471 slots.push_back(Slot);
484 unsigned StackColoring::collectMarkers(unsigned NumSlot)
486 unsigned MarkersFound = 0;
487 BlockBitVecMap SeenStartMap;
488 InterestingSlots.clear();
489 InterestingSlots.resize(NumSlot);
490 ConservativeSlots.clear();
491 ConservativeSlots.resize(NumSlot);
493 // number of start and end lifetime ops for each slot
494 SmallVector<int, 8> NumStartLifetimes(NumSlot, 0);
495 SmallVector<int, 8> NumEndLifetimes(NumSlot, 0);
497 // Step 1: collect markers and populate the "InterestingSlots"
498 // and "ConservativeSlots" sets.
499 for (MachineBasicBlock *MBB : depth_first(MF)) {
501 // Compute the set of slots for which we've seen a START marker but have
502 // not yet seen an END marker at this point in the walk (e.g. on entry
504 BitVector BetweenStartEnd;
505 BetweenStartEnd.resize(NumSlot);
506 for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
507 PE = MBB->pred_end(); PI != PE; ++PI) {
508 BlockBitVecMap::const_iterator I = SeenStartMap.find(*PI);
509 if (I != SeenStartMap.end()) {
510 BetweenStartEnd |= I->second;
514 // Walk the instructions in the block to look for start/end ops.
515 for (MachineInstr &MI : *MBB) {
516 if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
517 MI.getOpcode() == TargetOpcode::LIFETIME_END) {
518 int Slot = getStartOrEndSlot(MI);
521 InterestingSlots.set(Slot);
522 if (MI.getOpcode() == TargetOpcode::LIFETIME_START) {
523 BetweenStartEnd.set(Slot);
524 NumStartLifetimes[Slot] += 1;
526 BetweenStartEnd.reset(Slot);
527 NumEndLifetimes[Slot] += 1;
529 const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
531 DEBUG(dbgs() << "Found a lifetime ");
532 DEBUG(dbgs() << (MI.getOpcode() == TargetOpcode::LIFETIME_START
535 DEBUG(dbgs() << " marker for slot #" << Slot);
536 DEBUG(dbgs() << " with allocation: " << Allocation->getName()
539 Markers.push_back(&MI);
542 for (const MachineOperand &MO : MI.operands()) {
545 int Slot = MO.getIndex();
548 if (! BetweenStartEnd.test(Slot)) {
549 ConservativeSlots.set(Slot);
554 BitVector &SeenStart = SeenStartMap[MBB];
555 SeenStart |= BetweenStartEnd;
561 // PR27903: slots with multiple start or end lifetime ops are not
562 // safe to enable for "lifetime-start-on-first-use".
563 for (unsigned slot = 0; slot < NumSlot; ++slot)
564 if (NumStartLifetimes[slot] > 1 || NumEndLifetimes[slot] > 1)
565 ConservativeSlots.set(slot);
566 DEBUG(dumpBV("Conservative slots", ConservativeSlots));
568 // Step 2: compute begin/end sets for each block
570 // NOTE: We use a depth-first iteration to ensure that we obtain a
571 // deterministic numbering.
572 for (MachineBasicBlock *MBB : depth_first(MF)) {
574 // Assign a serial number to this basic block.
575 BasicBlocks[MBB] = BasicBlockNumbering.size();
576 BasicBlockNumbering.push_back(MBB);
578 // Keep a reference to avoid repeated lookups.
579 BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
581 BlockInfo.Begin.resize(NumSlot);
582 BlockInfo.End.resize(NumSlot);
584 SmallVector<int, 4> slots;
585 for (MachineInstr &MI : *MBB) {
586 bool isStart = false;
588 if (isLifetimeStartOrEnd(MI, slots, isStart)) {
590 assert(slots.size() == 1 && "unexpected: MI ends multiple slots");
592 if (BlockInfo.Begin.test(Slot)) {
593 BlockInfo.Begin.reset(Slot);
595 BlockInfo.End.set(Slot);
597 for (auto Slot : slots) {
598 DEBUG(dbgs() << "Found a use of slot #" << Slot);
599 DEBUG(dbgs() << " at BB#" << MBB->getNumber() << " index ");
600 DEBUG(Indexes->getInstructionIndex(MI).print(dbgs()));
601 const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
603 DEBUG(dbgs() << " with allocation: "<< Allocation->getName());
605 DEBUG(dbgs() << "\n");
606 if (BlockInfo.End.test(Slot)) {
607 BlockInfo.End.reset(Slot);
609 BlockInfo.Begin.set(Slot);
616 // Update statistics.
617 NumMarkerSeen += MarkersFound;
621 void StackColoring::calculateLocalLiveness()
623 unsigned NumIters = 0;
629 for (const MachineBasicBlock *BB : BasicBlockNumbering) {
631 // Use an iterator to avoid repeated lookups.
632 LivenessMap::iterator BI = BlockLiveness.find(BB);
633 assert(BI != BlockLiveness.end() && "Block not found");
634 BlockLifetimeInfo &BlockInfo = BI->second;
636 // Compute LiveIn by unioning together the LiveOut sets of all preds.
637 BitVector LocalLiveIn;
638 for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
639 PE = BB->pred_end(); PI != PE; ++PI) {
640 LivenessMap::const_iterator I = BlockLiveness.find(*PI);
641 assert(I != BlockLiveness.end() && "Predecessor not found");
642 LocalLiveIn |= I->second.LiveOut;
645 // Compute LiveOut by subtracting out lifetimes that end in this
646 // block, then adding in lifetimes that begin in this block. If
647 // we have both BEGIN and END markers in the same basic block
648 // then we know that the BEGIN marker comes after the END,
649 // because we already handle the case where the BEGIN comes
650 // before the END when collecting the markers (and building the
651 // BEGIN/END vectors).
652 BitVector LocalLiveOut = LocalLiveIn;
653 LocalLiveOut.reset(BlockInfo.End);
654 LocalLiveOut |= BlockInfo.Begin;
656 // Update block LiveIn set, noting whether it has changed.
657 if (LocalLiveIn.test(BlockInfo.LiveIn)) {
659 BlockInfo.LiveIn |= LocalLiveIn;
662 // Update block LiveOut set, noting whether it has changed.
663 if (LocalLiveOut.test(BlockInfo.LiveOut)) {
665 BlockInfo.LiveOut |= LocalLiveOut;
670 NumIterations = NumIters;
673 void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
674 SmallVector<SlotIndex, 16> Starts;
675 SmallVector<SlotIndex, 16> Finishes;
677 // For each block, find which slots are active within this block
678 // and update the live intervals.
679 for (const MachineBasicBlock &MBB : *MF) {
681 Starts.resize(NumSlots);
683 Finishes.resize(NumSlots);
685 // Create the interval for the basic blocks containing lifetime begin/end.
686 for (const MachineInstr &MI : MBB) {
688 SmallVector<int, 4> slots;
689 bool IsStart = false;
690 if (!isLifetimeStartOrEnd(MI, slots, IsStart))
692 SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
693 for (auto Slot : slots) {
695 if (!Starts[Slot].isValid() || Starts[Slot] > ThisIndex)
696 Starts[Slot] = ThisIndex;
698 if (!Finishes[Slot].isValid() || Finishes[Slot] < ThisIndex)
699 Finishes[Slot] = ThisIndex;
704 // Create the interval of the blocks that we previously found to be 'alive'.
705 BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
706 for (unsigned pos : MBBLiveness.LiveIn.set_bits()) {
707 Starts[pos] = Indexes->getMBBStartIdx(&MBB);
709 for (unsigned pos : MBBLiveness.LiveOut.set_bits()) {
710 Finishes[pos] = Indexes->getMBBEndIdx(&MBB);
713 for (unsigned i = 0; i < NumSlots; ++i) {
715 // When LifetimeStartOnFirstUse is turned on, data flow analysis
716 // is forward (from starts to ends), not bidirectional. A
717 // consequence of this is that we can wind up in situations
718 // where Starts[i] is invalid but Finishes[i] is valid and vice
730 // Here the slot for "x" will not be live into the block
731 // containing the "return 2" (since lifetimes start with first
732 // use, not at the dominating LIFETIME_START marker).
734 if (Starts[i].isValid() && !Finishes[i].isValid()) {
735 Finishes[i] = Indexes->getMBBEndIdx(&MBB);
737 if (!Starts[i].isValid())
740 assert(Starts[i] && Finishes[i] && "Invalid interval");
741 VNInfo *ValNum = Intervals[i]->getValNumInfo(0);
742 SlotIndex S = Starts[i];
743 SlotIndex F = Finishes[i];
745 // We have a single consecutive region.
746 Intervals[i]->addSegment(LiveInterval::Segment(S, F, ValNum));
748 // We have two non-consecutive regions. This happens when
749 // LIFETIME_START appears after the LIFETIME_END marker.
750 SlotIndex NewStart = Indexes->getMBBStartIdx(&MBB);
751 SlotIndex NewFin = Indexes->getMBBEndIdx(&MBB);
752 Intervals[i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum));
753 Intervals[i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum));
759 bool StackColoring::removeAllMarkers() {
761 for (MachineInstr *MI : Markers) {
762 MI->eraseFromParent();
767 DEBUG(dbgs()<<"Removed "<<Count<<" markers.\n");
771 void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
772 unsigned FixedInstr = 0;
773 unsigned FixedMemOp = 0;
774 unsigned FixedDbg = 0;
776 // Remap debug information that refers to stack slots.
777 for (auto &VI : MF->getVariableDbgInfo()) {
780 if (SlotRemap.count(VI.Slot)) {
781 DEBUG(dbgs() << "Remapping debug info for ["
782 << cast<DILocalVariable>(VI.Var)->getName() << "].\n");
783 VI.Slot = SlotRemap[VI.Slot];
788 // Keep a list of *allocas* which need to be remapped.
789 DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
790 for (const std::pair<int, int> &SI : SlotRemap) {
791 const AllocaInst *From = MFI->getObjectAllocation(SI.first);
792 const AllocaInst *To = MFI->getObjectAllocation(SI.second);
793 assert(To && From && "Invalid allocation object");
796 // AA might be used later for instruction scheduling, and we need it to be
797 // able to deduce the correct aliasing releationships between pointers
798 // derived from the alloca being remapped and the target of that remapping.
799 // The only safe way, without directly informing AA about the remapping
800 // somehow, is to directly update the IR to reflect the change being made
802 Instruction *Inst = const_cast<AllocaInst *>(To);
803 if (From->getType() != To->getType()) {
804 BitCastInst *Cast = new BitCastInst(Inst, From->getType());
805 Cast->insertAfter(Inst);
809 // Allow the stack protector to adjust its value map to account for the
810 // upcoming replacement.
811 SP->adjustForColoring(From, To);
813 // The new alloca might not be valid in a llvm.dbg.declare for this
814 // variable, so undef out the use to make the verifier happy.
815 AllocaInst *FromAI = const_cast<AllocaInst *>(From);
816 if (FromAI->isUsedByMetadata())
817 ValueAsMetadata::handleRAUW(FromAI, UndefValue::get(FromAI->getType()));
818 for (auto &Use : FromAI->uses()) {
819 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
820 if (BCI->isUsedByMetadata())
821 ValueAsMetadata::handleRAUW(BCI, UndefValue::get(BCI->getType()));
824 // Note that this will not replace uses in MMOs (which we'll update below),
825 // or anywhere else (which is why we won't delete the original
827 FromAI->replaceAllUsesWith(Inst);
830 // Remap all instructions to the new stack slots.
831 for (MachineBasicBlock &BB : *MF)
832 for (MachineInstr &I : BB) {
833 // Skip lifetime markers. We'll remove them soon.
834 if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
835 I.getOpcode() == TargetOpcode::LIFETIME_END)
838 // Update the MachineMemOperand to use the new alloca.
839 for (MachineMemOperand *MMO : I.memoperands()) {
840 // FIXME: In order to enable the use of TBAA when using AA in CodeGen,
841 // we'll also need to update the TBAA nodes in MMOs with values
842 // derived from the merged allocas. When doing this, we'll need to use
843 // the same variant of GetUnderlyingObjects that is used by the
844 // instruction scheduler (that can look through ptrtoint/inttoptr
847 // We've replaced IR-level uses of the remapped allocas, so we only
848 // need to replace direct uses here.
849 const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
853 if (!Allocas.count(AI))
856 MMO->setValue(Allocas[AI]);
860 // Update all of the machine instruction operands.
861 for (MachineOperand &MO : I.operands()) {
864 int FromSlot = MO.getIndex();
866 // Don't touch arguments.
870 // Only look at mapped slots.
871 if (!SlotRemap.count(FromSlot))
874 // In a debug build, check that the instruction that we are modifying is
875 // inside the expected live range. If the instruction is not inside
876 // the calculated range then it means that the alloca usage moved
877 // outside of the lifetime markers, or that the user has a bug.
878 // NOTE: Alloca address calculations which happen outside the lifetime
879 // zone are are okay, despite the fact that we don't have a good way
880 // for validating all of the usages of the calculation.
882 bool TouchesMemory = I.mayLoad() || I.mayStore();
883 // If we *don't* protect the user from escaped allocas, don't bother
884 // validating the instructions.
885 if (!I.isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
886 SlotIndex Index = Indexes->getInstructionIndex(I);
887 const LiveInterval *Interval = &*Intervals[FromSlot];
888 assert(Interval->find(Index) != Interval->end() &&
889 "Found instruction usage outside of live range.");
893 // Fix the machine instructions.
894 int ToSlot = SlotRemap[FromSlot];
900 // Update the location of C++ catch objects for the MSVC personality routine.
901 if (WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo())
902 for (WinEHTryBlockMapEntry &TBME : EHInfo->TryBlockMap)
903 for (WinEHHandlerType &H : TBME.HandlerArray)
904 if (H.CatchObj.FrameIndex != INT_MAX &&
905 SlotRemap.count(H.CatchObj.FrameIndex))
906 H.CatchObj.FrameIndex = SlotRemap[H.CatchObj.FrameIndex];
908 DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
909 DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
910 DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
913 void StackColoring::removeInvalidSlotRanges() {
914 for (MachineBasicBlock &BB : *MF)
915 for (MachineInstr &I : BB) {
916 if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
917 I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugValue())
920 // Some intervals are suspicious! In some cases we find address
921 // calculations outside of the lifetime zone, but not actual memory
922 // read or write. Memory accesses outside of the lifetime zone are a clear
923 // violation, but address calculations are okay. This can happen when
924 // GEPs are hoisted outside of the lifetime zone.
925 // So, in here we only check instructions which can read or write memory.
926 if (!I.mayLoad() && !I.mayStore())
929 // Check all of the machine operands.
930 for (const MachineOperand &MO : I.operands()) {
934 int Slot = MO.getIndex();
939 if (Intervals[Slot]->empty())
942 // Check that the used slot is inside the calculated lifetime range.
943 // If it is not, warn about it and invalidate the range.
944 LiveInterval *Interval = &*Intervals[Slot];
945 SlotIndex Index = Indexes->getInstructionIndex(I);
946 if (Interval->find(Index) == Interval->end()) {
948 DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
955 void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
957 // Expunge slot remap map.
958 for (unsigned i=0; i < NumSlots; ++i) {
959 // If we are remapping i
960 if (SlotRemap.count(i)) {
961 int Target = SlotRemap[i];
962 // As long as our target is mapped to something else, follow it.
963 while (SlotRemap.count(Target)) {
964 Target = SlotRemap[Target];
965 SlotRemap[i] = Target;
971 bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
972 DEBUG(dbgs() << "********** Stack Coloring **********\n"
973 << "********** Function: "
974 << ((const Value*)Func.getFunction())->getName() << '\n');
976 MFI = &MF->getFrameInfo();
977 Indexes = &getAnalysis<SlotIndexes>();
978 SP = &getAnalysis<StackProtector>();
979 BlockLiveness.clear();
981 BasicBlockNumbering.clear();
984 VNInfoAllocator.Reset();
986 unsigned NumSlots = MFI->getObjectIndexEnd();
988 // If there are no stack slots then there are no markers to remove.
992 SmallVector<int, 8> SortedSlots;
993 SortedSlots.reserve(NumSlots);
994 Intervals.reserve(NumSlots);
996 unsigned NumMarkers = collectMarkers(NumSlots);
998 unsigned TotalSize = 0;
999 DEBUG(dbgs()<<"Found "<<NumMarkers<<" markers and "<<NumSlots<<" slots\n");
1000 DEBUG(dbgs()<<"Slot structure:\n");
1002 for (int i=0; i < MFI->getObjectIndexEnd(); ++i) {
1003 DEBUG(dbgs()<<"Slot #"<<i<<" - "<<MFI->getObjectSize(i)<<" bytes.\n");
1004 TotalSize += MFI->getObjectSize(i);
1007 DEBUG(dbgs()<<"Total Stack size: "<<TotalSize<<" bytes\n\n");
1009 // Don't continue because there are not enough lifetime markers, or the
1010 // stack is too small, or we are told not to optimize the slots.
1011 if (NumMarkers < 2 || TotalSize < 16 || DisableColoring ||
1012 skipFunction(*Func.getFunction())) {
1013 DEBUG(dbgs()<<"Will not try to merge slots.\n");
1014 return removeAllMarkers();
1017 for (unsigned i=0; i < NumSlots; ++i) {
1018 std::unique_ptr<LiveInterval> LI(new LiveInterval(i, 0));
1019 LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
1020 Intervals.push_back(std::move(LI));
1021 SortedSlots.push_back(i);
1024 // Calculate the liveness of each block.
1025 calculateLocalLiveness();
1026 DEBUG(dbgs() << "Dataflow iterations: " << NumIterations << "\n");
1029 // Propagate the liveness information.
1030 calculateLiveIntervals(NumSlots);
1031 DEBUG(dumpIntervals());
1033 // Search for allocas which are used outside of the declared lifetime
1035 if (ProtectFromEscapedAllocas)
1036 removeInvalidSlotRanges();
1038 // Maps old slots to new slots.
1039 DenseMap<int, int> SlotRemap;
1040 unsigned RemovedSlots = 0;
1041 unsigned ReducedSize = 0;
1043 // Do not bother looking at empty intervals.
1044 for (unsigned I = 0; I < NumSlots; ++I) {
1045 if (Intervals[SortedSlots[I]]->empty())
1046 SortedSlots[I] = -1;
1049 // This is a simple greedy algorithm for merging allocas. First, sort the
1050 // slots, placing the largest slots first. Next, perform an n^2 scan and look
1051 // for disjoint slots. When you find disjoint slots, merge the samller one
1052 // into the bigger one and update the live interval. Remove the small alloca
1055 // Sort the slots according to their size. Place unused slots at the end.
1056 // Use stable sort to guarantee deterministic code generation.
1057 std::stable_sort(SortedSlots.begin(), SortedSlots.end(),
1058 [this](int LHS, int RHS) {
1059 // We use -1 to denote a uninteresting slot. Place these slots at the end.
1060 if (LHS == -1) return false;
1061 if (RHS == -1) return true;
1062 // Sort according to size.
1063 return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
1066 bool Changed = true;
1069 for (unsigned I = 0; I < NumSlots; ++I) {
1070 if (SortedSlots[I] == -1)
1073 for (unsigned J=I+1; J < NumSlots; ++J) {
1074 if (SortedSlots[J] == -1)
1077 int FirstSlot = SortedSlots[I];
1078 int SecondSlot = SortedSlots[J];
1079 LiveInterval *First = &*Intervals[FirstSlot];
1080 LiveInterval *Second = &*Intervals[SecondSlot];
1081 assert (!First->empty() && !Second->empty() && "Found an empty range");
1083 // Merge disjoint slots.
1084 if (!First->overlaps(*Second)) {
1086 First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
1087 SlotRemap[SecondSlot] = FirstSlot;
1088 SortedSlots[J] = -1;
1089 DEBUG(dbgs()<<"Merging #"<<FirstSlot<<" and slots #"<<
1090 SecondSlot<<" together.\n");
1091 unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
1092 MFI->getObjectAlignment(SecondSlot));
1094 assert(MFI->getObjectSize(FirstSlot) >=
1095 MFI->getObjectSize(SecondSlot) &&
1096 "Merging a small object into a larger one");
1099 ReducedSize += MFI->getObjectSize(SecondSlot);
1100 MFI->setObjectAlignment(FirstSlot, MaxAlignment);
1101 MFI->RemoveStackObject(SecondSlot);
1107 // Record statistics.
1108 StackSpaceSaved += ReducedSize;
1109 StackSlotMerged += RemovedSlots;
1110 DEBUG(dbgs()<<"Merge "<<RemovedSlots<<" slots. Saved "<<
1111 ReducedSize<<" bytes\n");
1113 // Scan the entire function and update all machine operands that use frame
1114 // indices to use the remapped frame index.
1115 expungeSlotMap(SlotRemap, NumSlots);
1116 remapInstructions(SlotRemap);
1118 return removeAllMarkers();