1 //===-- StackColoring.cpp -------------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass implements the stack-coloring optimization that looks for
11 // lifetime markers machine instructions (LIFESTART_BEGIN and LIFESTART_END),
12 // which represent the possible lifetime of stack slots. It attempts to
13 // merge disjoint stack slots and reduce the used stack space.
14 // NOTE: This pass is not StackSlotColoring, which optimizes spill slots.
16 // TODO: In the future we plan to improve stack coloring in the following ways:
17 // 1. Allow merging multiple small slots into a single larger slot at different
19 // 2. Merge this pass with StackSlotColoring and allow merging of allocas with
22 //===----------------------------------------------------------------------===//
24 #include "llvm/ADT/BitVector.h"
25 #include "llvm/ADT/DepthFirstIterator.h"
26 #include "llvm/ADT/SetVector.h"
27 #include "llvm/ADT/SmallPtrSet.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/Analysis/ValueTracking.h"
30 #include "llvm/CodeGen/LiveInterval.h"
31 #include "llvm/CodeGen/MachineBasicBlock.h"
32 #include "llvm/CodeGen/MachineFrameInfo.h"
33 #include "llvm/CodeGen/MachineFunctionPass.h"
34 #include "llvm/CodeGen/MachineLoopInfo.h"
35 #include "llvm/CodeGen/MachineMemOperand.h"
36 #include "llvm/CodeGen/MachineModuleInfo.h"
37 #include "llvm/CodeGen/MachineRegisterInfo.h"
38 #include "llvm/CodeGen/Passes.h"
39 #include "llvm/CodeGen/PseudoSourceValue.h"
40 #include "llvm/CodeGen/SlotIndexes.h"
41 #include "llvm/CodeGen/StackProtector.h"
42 #include "llvm/CodeGen/WinEHFuncInfo.h"
43 #include "llvm/IR/DebugInfo.h"
44 #include "llvm/IR/Function.h"
45 #include "llvm/IR/Instructions.h"
46 #include "llvm/IR/IntrinsicInst.h"
47 #include "llvm/IR/Module.h"
48 #include "llvm/Support/CommandLine.h"
49 #include "llvm/Support/Debug.h"
50 #include "llvm/Support/raw_ostream.h"
51 #include "llvm/Target/TargetInstrInfo.h"
52 #include "llvm/Target/TargetRegisterInfo.h"
56 #define DEBUG_TYPE "stack-coloring"
59 DisableColoring("no-stack-coloring",
60 cl::init(false), cl::Hidden,
61 cl::desc("Disable stack coloring"));
63 /// The user may write code that uses allocas outside of the declared lifetime
64 /// zone. This can happen when the user returns a reference to a local
65 /// data-structure. We can detect these cases and decide not to optimize the
66 /// code. If this flag is enabled, we try to save the user. This option
67 /// is treated as overriding LifetimeStartOnFirstUse below.
69 ProtectFromEscapedAllocas("protect-from-escaped-allocas",
70 cl::init(false), cl::Hidden,
71 cl::desc("Do not optimize lifetime zones that "
74 /// Enable enhanced dataflow scheme for lifetime analysis (treat first
75 /// use of stack slot as start of slot lifetime, as opposed to looking
76 /// for LIFETIME_START marker). See "Implementation notes" below for
79 LifetimeStartOnFirstUse("stackcoloring-lifetime-start-on-first-use",
80 cl::init(true), cl::Hidden,
81 cl::desc("Treat stack lifetimes as starting on first use, not on START marker."));
84 STATISTIC(NumMarkerSeen, "Number of lifetime markers found.");
85 STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
86 STATISTIC(StackSlotMerged, "Number of stack slot merged.");
87 STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
89 //===----------------------------------------------------------------------===//
91 //===----------------------------------------------------------------------===//
93 // Stack Coloring reduces stack usage by merging stack slots when they
94 // can't be used together. For example, consider the following C program:
96 // void bar(char *, int);
97 // void foo(bool var) {
116 // Naively-compiled, this program would use 12k of stack space. However, the
117 // stack slot corresponding to `z` is always destroyed before either of the
118 // stack slots for `x` or `y` are used, and then `x` is only used if `var`
119 // is true, while `y` is only used if `var` is false. So in no time are 2
120 // of the stack slots used together, and therefore we can merge them,
121 // compiling the function using only a single 4k alloca:
123 // void foo(bool var) { // equivalent
136 // This is an important optimization if we want stack space to be under
137 // control in large functions, both open-coded ones and ones created by
140 // Implementation Notes:
141 // ---------------------
143 // An important part of the above reasoning is that `z` can't be accessed
144 // while the latter 2 calls to `bar` are running. This is justified because
145 // `z`'s lifetime is over after we exit from block `A:`, so any further
146 // accesses to it would be UB. The way we represent this information
147 // in LLVM is by having frontends delimit blocks with `lifetime.start`
148 // and `lifetime.end` intrinsics.
150 // The effect of these intrinsics seems to be as follows (maybe I should
151 // specify this in the reference?):
153 // L1) at start, each stack-slot is marked as *out-of-scope*, unless no
154 // lifetime intrinsic refers to that stack slot, in which case
155 // it is marked as *in-scope*.
156 // L2) on a `lifetime.start`, a stack slot is marked as *in-scope* and
157 // the stack slot is overwritten with `undef`.
158 // L3) on a `lifetime.end`, a stack slot is marked as *out-of-scope*.
159 // L4) on function exit, all stack slots are marked as *out-of-scope*.
160 // L5) `lifetime.end` is a no-op when called on a slot that is already
162 // L6) memory accesses to *out-of-scope* stack slots are UB.
163 // L7) when a stack-slot is marked as *out-of-scope*, all pointers to it
164 // are invalidated, unless the slot is "degenerate". This is used to
165 // justify not marking slots as in-use until the pointer to them is
166 // used, but feels a bit hacky in the presence of things like LICM. See
167 // the "Degenerate Slots" section for more details.
169 // Now, let's ground stack coloring on these rules. We'll define a slot
170 // as *in-use* at a (dynamic) point in execution if it either can be
171 // written to at that point, or if it has a live and non-undef content
174 // Obviously, slots that are never *in-use* together can be merged, and
175 // in our example `foo`, the slots for `x`, `y` and `z` are never
176 // in-use together (of course, sometimes slots that *are* in-use together
177 // might still be mergable, but we don't care about that here).
179 // In this implementation, we successively merge pairs of slots that are
180 // not *in-use* together. We could be smarter - for example, we could merge
181 // a single large slot with 2 small slots, or we could construct the
182 // interference graph and run a "smart" graph coloring algorithm, but with
183 // that aside, how do we find out whether a pair of slots might be *in-use*
186 // From our rules, we see that *out-of-scope* slots are never *in-use*,
187 // and from (L7) we see that "non-degenerate" slots remain non-*in-use*
188 // until their address is taken. Therefore, we can approximate slot activity
191 // A subtle point: naively, we might try to figure out which pairs of
192 // stack-slots interfere by propagating `S in-use` through the CFG for every
193 // stack-slot `S`, and having `S` and `T` interfere if there is a CFG point in
194 // which they are both *in-use*.
196 // That is sound, but overly conservative in some cases: in our (artificial)
197 // example `foo`, either `x` or `y` might be in use at the label `B:`, but
198 // as `x` is only in use if we came in from the `var` edge and `y` only
199 // if we came from the `!var` edge, they still can't be in use together.
200 // See PR32488 for an important real-life case.
202 // If we wanted to find all points of interference precisely, we could
203 // propagate `S in-use` and `S&T in-use` predicates through the CFG. That
204 // would be precise, but requires propagating `O(n^2)` dataflow facts.
206 // However, we aren't interested in the *set* of points of interference
207 // between 2 stack slots, only *whether* there *is* such a point. So we
208 // can rely on a little trick: for `S` and `T` to be in-use together,
209 // one of them needs to become in-use while the other is in-use (or
210 // they might both become in use simultaneously). We can check this
211 // by also keeping track of the points at which a stack slot might *start*
217 // Consider the following motivating example:
220 // char b1[1024], b2[1024];
226 // char b4[1024], b5[1024];
227 // <uses of b2, b4, b5>;
232 // In the code above, "b3" and "b4" are declared in distinct lexical
233 // scopes, meaning that it is easy to prove that they can share the
234 // same stack slot. Variables "b1" and "b2" are declared in the same
235 // scope, meaning that from a lexical point of view, their lifetimes
236 // overlap. From a control flow pointer of view, however, the two
237 // variables are accessed in disjoint regions of the CFG, thus it
238 // should be possible for them to share the same stack slot. An ideal
239 // stack allocation for the function above would look like:
245 // Achieving this allocation is tricky, however, due to the way
246 // lifetime markers are inserted. Here is a simplified view of the
247 // control flow graph for the code above:
249 // +------ block 0 -------+
250 // 0| LIFETIME_START b1, b2 |
251 // 1| <test 'if' condition> |
252 // +-----------------------+
254 // +------ block 1 -------+ +------ block 2 -------+
255 // 2| LIFETIME_START b3 | 5| LIFETIME_START b4, b5 |
256 // 3| <uses of b1, b3> | 6| <uses of b2, b4, b5> |
257 // 4| LIFETIME_END b3 | 7| LIFETIME_END b4, b5 |
258 // +-----------------------+ +-----------------------+
260 // +------ block 3 -------+
261 // 8| <cleanupcode> |
262 // 9| LIFETIME_END b1, b2 |
264 // +-----------------------+
266 // If we create live intervals for the variables above strictly based
267 // on the lifetime markers, we'll get the set of intervals on the
268 // left. If we ignore the lifetime start markers and instead treat a
269 // variable's lifetime as beginning with the first reference to the
270 // var, then we get the intervals on the right.
272 // LIFETIME_START First Use
273 // b1: [0,9] [3,4] [8,9]
279 // For the intervals on the left, the best we can do is overlap two
280 // variables (b3 and b4, for example); this gives us a stack size of
281 // 4*1024 bytes, not ideal. When treating first-use as the start of a
282 // lifetime, we can additionally overlap b1 and b5, giving us a 3*1024
283 // byte stack (better).
288 // Relying entirely on first-use of stack slots is problematic,
289 // however, due to the fact that optimizations can sometimes migrate
290 // uses of a variable outside of its lifetime start/end region. Here
294 // char b1[1024], b2[1024];
307 // Before optimization, the control flow graph for the code above
308 // might look like the following:
310 // +------ block 0 -------+
311 // 0| LIFETIME_START b1, b2 |
312 // 1| <test 'if' condition> |
313 // +-----------------------+
315 // +------ block 1 -------+ +------- block 2 -------+
316 // 2| <uses of b2> | 3| <uses of b1> |
317 // +-----------------------+ +-----------------------+
319 // | +------- block 3 -------+ <-\.
320 // | 4| <while condition> | |
321 // | +-----------------------+ |
323 // | / +------- block 4 -------+
324 // \ / 5| LIFETIME_START b3 | |
325 // \ / 6| <uses of b3> | |
326 // \ / 7| LIFETIME_END b3 | |
327 // \ | +------------------------+ |
329 // +------ block 5 -----+ \---------------
330 // 8| <cleanupcode> |
331 // 9| LIFETIME_END b1, b2 |
333 // +---------------------+
335 // During optimization, however, it can happen that an instruction
336 // computing an address in "b3" (for example, a loop-invariant GEP) is
337 // hoisted up out of the loop from block 4 to block 2. [Note that
338 // this is not an actual load from the stack, only an instruction that
339 // computes the address to be loaded]. If this happens, there is now a
340 // path leading from the first use of b3 to the return instruction
341 // that does not encounter the b3 LIFETIME_END, hence b3's lifetime is
342 // now larger than if we were computing live intervals strictly based
343 // on lifetime markers. In the example above, this lengthened lifetime
344 // would mean that it would appear illegal to overlap b3 with b2.
346 // To deal with this such cases, the code in ::collectMarkers() below
347 // tries to identify "degenerate" slots -- those slots where on a single
348 // forward pass through the CFG we encounter a first reference to slot
349 // K before we hit the slot K lifetime start marker. For such slots,
350 // we fall back on using the lifetime start marker as the beginning of
351 // the variable's lifetime. NB: with this implementation, slots can
352 // appear degenerate in cases where there is unstructured control flow:
357 // memcpy(&b[0], ...);
362 // If in RPO ordering chosen to walk the CFG we happen to visit the b[k]
363 // before visiting the memcpy block (which will contain the lifetime start
364 // for "b" then it will appear that 'b' has a degenerate lifetime.
368 /// StackColoring - A machine pass for merging disjoint stack allocations,
369 /// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
370 class StackColoring : public MachineFunctionPass {
371 MachineFrameInfo *MFI;
374 /// A class representing liveness information for a single basic block.
375 /// Each bit in the BitVector represents the liveness property
376 /// for a different stack slot.
377 struct BlockLifetimeInfo {
378 /// Which slots BEGINs in each basic block.
380 /// Which slots ENDs in each basic block.
382 /// Which slots are marked as LIVE_IN, coming into each basic block.
384 /// Which slots are marked as LIVE_OUT, coming out of each basic block.
388 /// Maps active slots (per bit) for each basic block.
389 typedef DenseMap<const MachineBasicBlock*, BlockLifetimeInfo> LivenessMap;
390 LivenessMap BlockLiveness;
392 /// Maps serial numbers to basic blocks.
393 DenseMap<const MachineBasicBlock*, int> BasicBlocks;
394 /// Maps basic blocks to a serial number.
395 SmallVector<const MachineBasicBlock*, 8> BasicBlockNumbering;
397 /// Maps slots to their use interval. Outside of this interval, slots
398 /// values are either dead or `undef` and they will not be written to.
399 SmallVector<std::unique_ptr<LiveInterval>, 16> Intervals;
400 /// Maps slots to the points where they can become in-use.
401 SmallVector<SmallVector<SlotIndex, 4>, 16> LiveStarts;
402 /// VNInfo is used for the construction of LiveIntervals.
403 VNInfo::Allocator VNInfoAllocator;
404 /// SlotIndex analysis object.
405 SlotIndexes *Indexes;
406 /// The stack protector object.
409 /// The list of lifetime markers found. These markers are to be removed
410 /// once the coloring is done.
411 SmallVector<MachineInstr*, 8> Markers;
413 /// Record the FI slots for which we have seen some sort of
414 /// lifetime marker (either start or end).
415 BitVector InterestingSlots;
417 /// FI slots that need to be handled conservatively (for these
418 /// slots lifetime-start-on-first-use is disabled).
419 BitVector ConservativeSlots;
421 /// Number of iterations taken during data flow analysis.
422 unsigned NumIterations;
426 StackColoring() : MachineFunctionPass(ID) {
427 initializeStackColoringPass(*PassRegistry::getPassRegistry());
429 void getAnalysisUsage(AnalysisUsage &AU) const override;
430 bool runOnMachineFunction(MachineFunction &MF) override;
435 void dumpIntervals() const;
436 void dumpBB(MachineBasicBlock *MBB) const;
437 void dumpBV(const char *tag, const BitVector &BV) const;
439 /// Removes all of the lifetime marker instructions from the function.
440 /// \returns true if any markers were removed.
441 bool removeAllMarkers();
443 /// Scan the machine function and find all of the lifetime markers.
444 /// Record the findings in the BEGIN and END vectors.
445 /// \returns the number of markers found.
446 unsigned collectMarkers(unsigned NumSlot);
448 /// Perform the dataflow calculation and calculate the lifetime for each of
449 /// the slots, based on the BEGIN/END vectors. Set the LifetimeLIVE_IN and
450 /// LifetimeLIVE_OUT maps that represent which stack slots are live coming
451 /// in and out blocks.
452 void calculateLocalLiveness();
454 /// Returns TRUE if we're using the first-use-begins-lifetime method for
455 /// this slot (if FALSE, then the start marker is treated as start of lifetime).
456 bool applyFirstUse(int Slot) {
457 if (!LifetimeStartOnFirstUse || ProtectFromEscapedAllocas)
459 if (ConservativeSlots.test(Slot))
464 /// Examines the specified instruction and returns TRUE if the instruction
465 /// represents the start or end of an interesting lifetime. The slot or slots
466 /// starting or ending are added to the vector "slots" and "isStart" is set
468 /// \returns True if inst contains a lifetime start or end
469 bool isLifetimeStartOrEnd(const MachineInstr &MI,
470 SmallVector<int, 4> &slots,
473 /// Construct the LiveIntervals for the slots.
474 void calculateLiveIntervals(unsigned NumSlots);
476 /// Go over the machine function and change instructions which use stack
477 /// slots to use the joint slots.
478 void remapInstructions(DenseMap<int, int> &SlotRemap);
480 /// The input program may contain instructions which are not inside lifetime
481 /// markers. This can happen due to a bug in the compiler or due to a bug in
482 /// user code (for example, returning a reference to a local variable).
483 /// This procedure checks all of the instructions in the function and
484 /// invalidates lifetime ranges which do not contain all of the instructions
485 /// which access that frame slot.
486 void removeInvalidSlotRanges();
488 /// Map entries which point to other entries to their destination.
489 /// A->B->C becomes A->C.
490 void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
492 /// Used in collectMarkers
493 typedef DenseMap<const MachineBasicBlock*, BitVector> BlockBitVecMap;
495 } // end anonymous namespace
497 char StackColoring::ID = 0;
498 char &llvm::StackColoringID = StackColoring::ID;
500 INITIALIZE_PASS_BEGIN(StackColoring, DEBUG_TYPE,
501 "Merge disjoint stack slots", false, false)
502 INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
503 INITIALIZE_PASS_DEPENDENCY(StackProtector)
504 INITIALIZE_PASS_END(StackColoring, DEBUG_TYPE,
505 "Merge disjoint stack slots", false, false)
507 void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
508 AU.addRequired<SlotIndexes>();
509 AU.addRequired<StackProtector>();
510 MachineFunctionPass::getAnalysisUsage(AU);
513 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
514 LLVM_DUMP_METHOD void StackColoring::dumpBV(const char *tag,
515 const BitVector &BV) const {
516 dbgs() << tag << " : { ";
517 for (unsigned I = 0, E = BV.size(); I != E; ++I)
518 dbgs() << BV.test(I) << " ";
522 LLVM_DUMP_METHOD void StackColoring::dumpBB(MachineBasicBlock *MBB) const {
523 LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
524 assert(BI != BlockLiveness.end() && "Block not found");
525 const BlockLifetimeInfo &BlockInfo = BI->second;
527 dumpBV("BEGIN", BlockInfo.Begin);
528 dumpBV("END", BlockInfo.End);
529 dumpBV("LIVE_IN", BlockInfo.LiveIn);
530 dumpBV("LIVE_OUT", BlockInfo.LiveOut);
533 LLVM_DUMP_METHOD void StackColoring::dump() const {
534 for (MachineBasicBlock *MBB : depth_first(MF)) {
535 dbgs() << "Inspecting block #" << MBB->getNumber() << " ["
536 << MBB->getName() << "]\n";
541 LLVM_DUMP_METHOD void StackColoring::dumpIntervals() const {
542 for (unsigned I = 0, E = Intervals.size(); I != E; ++I) {
543 dbgs() << "Interval[" << I << "]:\n";
544 Intervals[I]->dump();
549 static inline int getStartOrEndSlot(const MachineInstr &MI)
551 assert((MI.getOpcode() == TargetOpcode::LIFETIME_START ||
552 MI.getOpcode() == TargetOpcode::LIFETIME_END) &&
553 "Expected LIFETIME_START or LIFETIME_END op");
554 const MachineOperand &MO = MI.getOperand(0);
555 int Slot = MO.getIndex();
562 // At the moment the only way to end a variable lifetime is with
563 // a VARIABLE_LIFETIME op (which can't contain a start). If things
564 // change and the IR allows for a single inst that both begins
565 // and ends lifetime(s), this interface will need to be reworked.
567 bool StackColoring::isLifetimeStartOrEnd(const MachineInstr &MI,
568 SmallVector<int, 4> &slots,
571 if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
572 MI.getOpcode() == TargetOpcode::LIFETIME_END) {
573 int Slot = getStartOrEndSlot(MI);
576 if (!InterestingSlots.test(Slot))
578 slots.push_back(Slot);
579 if (MI.getOpcode() == TargetOpcode::LIFETIME_END) {
583 if (! applyFirstUse(Slot)) {
587 } else if (LifetimeStartOnFirstUse && !ProtectFromEscapedAllocas) {
588 if (! MI.isDebugValue()) {
590 for (const MachineOperand &MO : MI.operands()) {
593 int Slot = MO.getIndex();
596 if (InterestingSlots.test(Slot) && applyFirstUse(Slot)) {
597 slots.push_back(Slot);
610 unsigned StackColoring::collectMarkers(unsigned NumSlot)
612 unsigned MarkersFound = 0;
613 BlockBitVecMap SeenStartMap;
614 InterestingSlots.clear();
615 InterestingSlots.resize(NumSlot);
616 ConservativeSlots.clear();
617 ConservativeSlots.resize(NumSlot);
619 // number of start and end lifetime ops for each slot
620 SmallVector<int, 8> NumStartLifetimes(NumSlot, 0);
621 SmallVector<int, 8> NumEndLifetimes(NumSlot, 0);
623 // Step 1: collect markers and populate the "InterestingSlots"
624 // and "ConservativeSlots" sets.
625 for (MachineBasicBlock *MBB : depth_first(MF)) {
627 // Compute the set of slots for which we've seen a START marker but have
628 // not yet seen an END marker at this point in the walk (e.g. on entry
630 BitVector BetweenStartEnd;
631 BetweenStartEnd.resize(NumSlot);
632 for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
633 PE = MBB->pred_end(); PI != PE; ++PI) {
634 BlockBitVecMap::const_iterator I = SeenStartMap.find(*PI);
635 if (I != SeenStartMap.end()) {
636 BetweenStartEnd |= I->second;
640 // Walk the instructions in the block to look for start/end ops.
641 for (MachineInstr &MI : *MBB) {
642 if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
643 MI.getOpcode() == TargetOpcode::LIFETIME_END) {
644 int Slot = getStartOrEndSlot(MI);
647 InterestingSlots.set(Slot);
648 if (MI.getOpcode() == TargetOpcode::LIFETIME_START) {
649 BetweenStartEnd.set(Slot);
650 NumStartLifetimes[Slot] += 1;
652 BetweenStartEnd.reset(Slot);
653 NumEndLifetimes[Slot] += 1;
655 const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
657 DEBUG(dbgs() << "Found a lifetime ");
658 DEBUG(dbgs() << (MI.getOpcode() == TargetOpcode::LIFETIME_START
661 DEBUG(dbgs() << " marker for slot #" << Slot);
662 DEBUG(dbgs() << " with allocation: " << Allocation->getName()
665 Markers.push_back(&MI);
668 for (const MachineOperand &MO : MI.operands()) {
671 int Slot = MO.getIndex();
674 if (! BetweenStartEnd.test(Slot)) {
675 ConservativeSlots.set(Slot);
680 BitVector &SeenStart = SeenStartMap[MBB];
681 SeenStart |= BetweenStartEnd;
687 // PR27903: slots with multiple start or end lifetime ops are not
688 // safe to enable for "lifetime-start-on-first-use".
689 for (unsigned slot = 0; slot < NumSlot; ++slot)
690 if (NumStartLifetimes[slot] > 1 || NumEndLifetimes[slot] > 1)
691 ConservativeSlots.set(slot);
692 DEBUG(dumpBV("Conservative slots", ConservativeSlots));
694 // Step 2: compute begin/end sets for each block
696 // NOTE: We use a depth-first iteration to ensure that we obtain a
697 // deterministic numbering.
698 for (MachineBasicBlock *MBB : depth_first(MF)) {
700 // Assign a serial number to this basic block.
701 BasicBlocks[MBB] = BasicBlockNumbering.size();
702 BasicBlockNumbering.push_back(MBB);
704 // Keep a reference to avoid repeated lookups.
705 BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
707 BlockInfo.Begin.resize(NumSlot);
708 BlockInfo.End.resize(NumSlot);
710 SmallVector<int, 4> slots;
711 for (MachineInstr &MI : *MBB) {
712 bool isStart = false;
714 if (isLifetimeStartOrEnd(MI, slots, isStart)) {
716 assert(slots.size() == 1 && "unexpected: MI ends multiple slots");
718 if (BlockInfo.Begin.test(Slot)) {
719 BlockInfo.Begin.reset(Slot);
721 BlockInfo.End.set(Slot);
723 for (auto Slot : slots) {
724 DEBUG(dbgs() << "Found a use of slot #" << Slot);
725 DEBUG(dbgs() << " at BB#" << MBB->getNumber() << " index ");
726 DEBUG(Indexes->getInstructionIndex(MI).print(dbgs()));
727 const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
729 DEBUG(dbgs() << " with allocation: "<< Allocation->getName());
731 DEBUG(dbgs() << "\n");
732 if (BlockInfo.End.test(Slot)) {
733 BlockInfo.End.reset(Slot);
735 BlockInfo.Begin.set(Slot);
742 // Update statistics.
743 NumMarkerSeen += MarkersFound;
747 void StackColoring::calculateLocalLiveness()
749 unsigned NumIters = 0;
755 for (const MachineBasicBlock *BB : BasicBlockNumbering) {
757 // Use an iterator to avoid repeated lookups.
758 LivenessMap::iterator BI = BlockLiveness.find(BB);
759 assert(BI != BlockLiveness.end() && "Block not found");
760 BlockLifetimeInfo &BlockInfo = BI->second;
762 // Compute LiveIn by unioning together the LiveOut sets of all preds.
763 BitVector LocalLiveIn;
764 for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
765 PE = BB->pred_end(); PI != PE; ++PI) {
766 LivenessMap::const_iterator I = BlockLiveness.find(*PI);
767 assert(I != BlockLiveness.end() && "Predecessor not found");
768 LocalLiveIn |= I->second.LiveOut;
771 // Compute LiveOut by subtracting out lifetimes that end in this
772 // block, then adding in lifetimes that begin in this block. If
773 // we have both BEGIN and END markers in the same basic block
774 // then we know that the BEGIN marker comes after the END,
775 // because we already handle the case where the BEGIN comes
776 // before the END when collecting the markers (and building the
777 // BEGIN/END vectors).
778 BitVector LocalLiveOut = LocalLiveIn;
779 LocalLiveOut.reset(BlockInfo.End);
780 LocalLiveOut |= BlockInfo.Begin;
782 // Update block LiveIn set, noting whether it has changed.
783 if (LocalLiveIn.test(BlockInfo.LiveIn)) {
785 BlockInfo.LiveIn |= LocalLiveIn;
788 // Update block LiveOut set, noting whether it has changed.
789 if (LocalLiveOut.test(BlockInfo.LiveOut)) {
791 BlockInfo.LiveOut |= LocalLiveOut;
796 NumIterations = NumIters;
799 void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
800 SmallVector<SlotIndex, 16> Starts;
801 SmallVector<bool, 16> DefinitelyInUse;
803 // For each block, find which slots are active within this block
804 // and update the live intervals.
805 for (const MachineBasicBlock &MBB : *MF) {
807 Starts.resize(NumSlots);
808 DefinitelyInUse.clear();
809 DefinitelyInUse.resize(NumSlots);
811 // Start the interval of the slots that we previously found to be 'in-use'.
812 BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
813 for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
814 pos = MBBLiveness.LiveIn.find_next(pos)) {
815 Starts[pos] = Indexes->getMBBStartIdx(&MBB);
818 // Create the interval for the basic blocks containing lifetime begin/end.
819 for (const MachineInstr &MI : MBB) {
821 SmallVector<int, 4> slots;
822 bool IsStart = false;
823 if (!isLifetimeStartOrEnd(MI, slots, IsStart))
825 SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
826 for (auto Slot : slots) {
828 // If a slot is already definitely in use, we don't have to emit
829 // a new start marker because there is already a pre-existing
831 if (!DefinitelyInUse[Slot]) {
832 LiveStarts[Slot].push_back(ThisIndex);
833 DefinitelyInUse[Slot] = true;
835 if (!Starts[Slot].isValid())
836 Starts[Slot] = ThisIndex;
838 if (Starts[Slot].isValid()) {
839 VNInfo *VNI = Intervals[Slot]->getValNumInfo(0);
840 Intervals[Slot]->addSegment(
841 LiveInterval::Segment(Starts[Slot], ThisIndex, VNI));
842 Starts[Slot] = SlotIndex(); // Invalidate the start index
843 DefinitelyInUse[Slot] = false;
849 // Finish up started segments
850 for (unsigned i = 0; i < NumSlots; ++i) {
851 if (!Starts[i].isValid())
854 SlotIndex EndIdx = Indexes->getMBBEndIdx(&MBB);
855 VNInfo *VNI = Intervals[i]->getValNumInfo(0);
856 Intervals[i]->addSegment(LiveInterval::Segment(Starts[i], EndIdx, VNI));
861 bool StackColoring::removeAllMarkers() {
863 for (MachineInstr *MI : Markers) {
864 MI->eraseFromParent();
869 DEBUG(dbgs()<<"Removed "<<Count<<" markers.\n");
873 void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
874 unsigned FixedInstr = 0;
875 unsigned FixedMemOp = 0;
876 unsigned FixedDbg = 0;
878 // Remap debug information that refers to stack slots.
879 for (auto &VI : MF->getVariableDbgInfo()) {
882 if (SlotRemap.count(VI.Slot)) {
883 DEBUG(dbgs() << "Remapping debug info for ["
884 << cast<DILocalVariable>(VI.Var)->getName() << "].\n");
885 VI.Slot = SlotRemap[VI.Slot];
890 // Keep a list of *allocas* which need to be remapped.
891 DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
892 for (const std::pair<int, int> &SI : SlotRemap) {
893 const AllocaInst *From = MFI->getObjectAllocation(SI.first);
894 const AllocaInst *To = MFI->getObjectAllocation(SI.second);
895 assert(To && From && "Invalid allocation object");
898 // AA might be used later for instruction scheduling, and we need it to be
899 // able to deduce the correct aliasing releationships between pointers
900 // derived from the alloca being remapped and the target of that remapping.
901 // The only safe way, without directly informing AA about the remapping
902 // somehow, is to directly update the IR to reflect the change being made
904 Instruction *Inst = const_cast<AllocaInst *>(To);
905 if (From->getType() != To->getType()) {
906 BitCastInst *Cast = new BitCastInst(Inst, From->getType());
907 Cast->insertAfter(Inst);
911 // Allow the stack protector to adjust its value map to account for the
912 // upcoming replacement.
913 SP->adjustForColoring(From, To);
915 // The new alloca might not be valid in a llvm.dbg.declare for this
916 // variable, so undef out the use to make the verifier happy.
917 AllocaInst *FromAI = const_cast<AllocaInst *>(From);
918 if (FromAI->isUsedByMetadata())
919 ValueAsMetadata::handleRAUW(FromAI, UndefValue::get(FromAI->getType()));
920 for (auto &Use : FromAI->uses()) {
921 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
922 if (BCI->isUsedByMetadata())
923 ValueAsMetadata::handleRAUW(BCI, UndefValue::get(BCI->getType()));
926 // Note that this will not replace uses in MMOs (which we'll update below),
927 // or anywhere else (which is why we won't delete the original
929 FromAI->replaceAllUsesWith(Inst);
932 // Remap all instructions to the new stack slots.
933 for (MachineBasicBlock &BB : *MF)
934 for (MachineInstr &I : BB) {
935 // Skip lifetime markers. We'll remove them soon.
936 if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
937 I.getOpcode() == TargetOpcode::LIFETIME_END)
940 // Update the MachineMemOperand to use the new alloca.
941 for (MachineMemOperand *MMO : I.memoperands()) {
942 // FIXME: In order to enable the use of TBAA when using AA in CodeGen,
943 // we'll also need to update the TBAA nodes in MMOs with values
944 // derived from the merged allocas. When doing this, we'll need to use
945 // the same variant of GetUnderlyingObjects that is used by the
946 // instruction scheduler (that can look through ptrtoint/inttoptr
949 // We've replaced IR-level uses of the remapped allocas, so we only
950 // need to replace direct uses here.
951 const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
955 if (!Allocas.count(AI))
958 MMO->setValue(Allocas[AI]);
962 // Update all of the machine instruction operands.
963 for (MachineOperand &MO : I.operands()) {
966 int FromSlot = MO.getIndex();
968 // Don't touch arguments.
972 // Only look at mapped slots.
973 if (!SlotRemap.count(FromSlot))
976 // In a debug build, check that the instruction that we are modifying is
977 // inside the expected live range. If the instruction is not inside
978 // the calculated range then it means that the alloca usage moved
979 // outside of the lifetime markers, or that the user has a bug.
980 // NOTE: Alloca address calculations which happen outside the lifetime
981 // zone are are okay, despite the fact that we don't have a good way
982 // for validating all of the usages of the calculation.
984 bool TouchesMemory = I.mayLoad() || I.mayStore();
985 // If we *don't* protect the user from escaped allocas, don't bother
986 // validating the instructions.
987 if (!I.isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
988 SlotIndex Index = Indexes->getInstructionIndex(I);
989 const LiveInterval *Interval = &*Intervals[FromSlot];
990 assert(Interval->find(Index) != Interval->end() &&
991 "Found instruction usage outside of live range.");
995 // Fix the machine instructions.
996 int ToSlot = SlotRemap[FromSlot];
1002 // Update the location of C++ catch objects for the MSVC personality routine.
1003 if (WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo())
1004 for (WinEHTryBlockMapEntry &TBME : EHInfo->TryBlockMap)
1005 for (WinEHHandlerType &H : TBME.HandlerArray)
1006 if (H.CatchObj.FrameIndex != INT_MAX &&
1007 SlotRemap.count(H.CatchObj.FrameIndex))
1008 H.CatchObj.FrameIndex = SlotRemap[H.CatchObj.FrameIndex];
1010 DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
1011 DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
1012 DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
1015 void StackColoring::removeInvalidSlotRanges() {
1016 for (MachineBasicBlock &BB : *MF)
1017 for (MachineInstr &I : BB) {
1018 if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
1019 I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugValue())
1022 // Some intervals are suspicious! In some cases we find address
1023 // calculations outside of the lifetime zone, but not actual memory
1024 // read or write. Memory accesses outside of the lifetime zone are a clear
1025 // violation, but address calculations are okay. This can happen when
1026 // GEPs are hoisted outside of the lifetime zone.
1027 // So, in here we only check instructions which can read or write memory.
1028 if (!I.mayLoad() && !I.mayStore())
1031 // Check all of the machine operands.
1032 for (const MachineOperand &MO : I.operands()) {
1036 int Slot = MO.getIndex();
1041 if (Intervals[Slot]->empty())
1044 // Check that the used slot is inside the calculated lifetime range.
1045 // If it is not, warn about it and invalidate the range.
1046 LiveInterval *Interval = &*Intervals[Slot];
1047 SlotIndex Index = Indexes->getInstructionIndex(I);
1048 if (Interval->find(Index) == Interval->end()) {
1050 DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
1057 void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
1058 unsigned NumSlots) {
1059 // Expunge slot remap map.
1060 for (unsigned i=0; i < NumSlots; ++i) {
1061 // If we are remapping i
1062 if (SlotRemap.count(i)) {
1063 int Target = SlotRemap[i];
1064 // As long as our target is mapped to something else, follow it.
1065 while (SlotRemap.count(Target)) {
1066 Target = SlotRemap[Target];
1067 SlotRemap[i] = Target;
1073 bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
1074 DEBUG(dbgs() << "********** Stack Coloring **********\n"
1075 << "********** Function: "
1076 << ((const Value*)Func.getFunction())->getName() << '\n');
1078 MFI = &MF->getFrameInfo();
1079 Indexes = &getAnalysis<SlotIndexes>();
1080 SP = &getAnalysis<StackProtector>();
1081 BlockLiveness.clear();
1082 BasicBlocks.clear();
1083 BasicBlockNumbering.clear();
1087 VNInfoAllocator.Reset();
1089 unsigned NumSlots = MFI->getObjectIndexEnd();
1091 // If there are no stack slots then there are no markers to remove.
1095 SmallVector<int, 8> SortedSlots;
1096 SortedSlots.reserve(NumSlots);
1097 Intervals.reserve(NumSlots);
1098 LiveStarts.resize(NumSlots);
1100 unsigned NumMarkers = collectMarkers(NumSlots);
1102 unsigned TotalSize = 0;
1103 DEBUG(dbgs()<<"Found "<<NumMarkers<<" markers and "<<NumSlots<<" slots\n");
1104 DEBUG(dbgs()<<"Slot structure:\n");
1106 for (int i=0; i < MFI->getObjectIndexEnd(); ++i) {
1107 DEBUG(dbgs()<<"Slot #"<<i<<" - "<<MFI->getObjectSize(i)<<" bytes.\n");
1108 TotalSize += MFI->getObjectSize(i);
1111 DEBUG(dbgs()<<"Total Stack size: "<<TotalSize<<" bytes\n\n");
1113 // Don't continue because there are not enough lifetime markers, or the
1114 // stack is too small, or we are told not to optimize the slots.
1115 if (NumMarkers < 2 || TotalSize < 16 || DisableColoring ||
1116 skipFunction(*Func.getFunction())) {
1117 DEBUG(dbgs()<<"Will not try to merge slots.\n");
1118 return removeAllMarkers();
1121 for (unsigned i=0; i < NumSlots; ++i) {
1122 std::unique_ptr<LiveInterval> LI(new LiveInterval(i, 0));
1123 LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
1124 Intervals.push_back(std::move(LI));
1125 SortedSlots.push_back(i);
1128 // Calculate the liveness of each block.
1129 calculateLocalLiveness();
1130 DEBUG(dbgs() << "Dataflow iterations: " << NumIterations << "\n");
1133 // Propagate the liveness information.
1134 calculateLiveIntervals(NumSlots);
1135 DEBUG(dumpIntervals());
1137 // Search for allocas which are used outside of the declared lifetime
1139 if (ProtectFromEscapedAllocas)
1140 removeInvalidSlotRanges();
1142 // Maps old slots to new slots.
1143 DenseMap<int, int> SlotRemap;
1144 unsigned RemovedSlots = 0;
1145 unsigned ReducedSize = 0;
1147 // Do not bother looking at empty intervals.
1148 for (unsigned I = 0; I < NumSlots; ++I) {
1149 if (Intervals[SortedSlots[I]]->empty())
1150 SortedSlots[I] = -1;
1153 // This is a simple greedy algorithm for merging allocas. First, sort the
1154 // slots, placing the largest slots first. Next, perform an n^2 scan and look
1155 // for disjoint slots. When you find disjoint slots, merge the samller one
1156 // into the bigger one and update the live interval. Remove the small alloca
1159 // Sort the slots according to their size. Place unused slots at the end.
1160 // Use stable sort to guarantee deterministic code generation.
1161 std::stable_sort(SortedSlots.begin(), SortedSlots.end(),
1162 [this](int LHS, int RHS) {
1163 // We use -1 to denote a uninteresting slot. Place these slots at the end.
1164 if (LHS == -1) return false;
1165 if (RHS == -1) return true;
1166 // Sort according to size.
1167 return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
1170 for (auto &s : LiveStarts)
1171 std::sort(s.begin(), s.end());
1173 bool Changed = true;
1176 for (unsigned I = 0; I < NumSlots; ++I) {
1177 if (SortedSlots[I] == -1)
1180 for (unsigned J=I+1; J < NumSlots; ++J) {
1181 if (SortedSlots[J] == -1)
1184 int FirstSlot = SortedSlots[I];
1185 int SecondSlot = SortedSlots[J];
1186 LiveInterval *First = &*Intervals[FirstSlot];
1187 LiveInterval *Second = &*Intervals[SecondSlot];
1188 auto &FirstS = LiveStarts[FirstSlot];
1189 auto &SecondS = LiveStarts[SecondSlot];
1190 assert (!First->empty() && !Second->empty() && "Found an empty range");
1192 // Merge disjoint slots. This is a little bit tricky - see the
1193 // Implementation Notes section for an explanation.
1194 if (!First->isLiveAtIndexes(SecondS) &&
1195 !Second->isLiveAtIndexes(FirstS)) {
1197 First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
1199 int OldSize = FirstS.size();
1200 FirstS.append(SecondS.begin(), SecondS.end());
1201 auto Mid = FirstS.begin() + OldSize;
1202 std::inplace_merge(FirstS.begin(), Mid, FirstS.end());
1204 SlotRemap[SecondSlot] = FirstSlot;
1205 SortedSlots[J] = -1;
1206 DEBUG(dbgs()<<"Merging #"<<FirstSlot<<" and slots #"<<
1207 SecondSlot<<" together.\n");
1208 unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
1209 MFI->getObjectAlignment(SecondSlot));
1211 assert(MFI->getObjectSize(FirstSlot) >=
1212 MFI->getObjectSize(SecondSlot) &&
1213 "Merging a small object into a larger one");
1216 ReducedSize += MFI->getObjectSize(SecondSlot);
1217 MFI->setObjectAlignment(FirstSlot, MaxAlignment);
1218 MFI->RemoveStackObject(SecondSlot);
1224 // Record statistics.
1225 StackSpaceSaved += ReducedSize;
1226 StackSlotMerged += RemovedSlots;
1227 DEBUG(dbgs()<<"Merge "<<RemovedSlots<<" slots. Saved "<<
1228 ReducedSize<<" bytes\n");
1230 // Scan the entire function and update all machine operands that use frame
1231 // indices to use the remapped frame index.
1232 expungeSlotMap(SlotRemap, NumSlots);
1233 remapInstructions(SlotRemap);
1235 return removeAllMarkers();