1 //===- SplitKit.cpp - Toolkit for splitting live ranges -------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the SplitAnalysis class as well as mutator functions for
11 // live range splitting.
13 //===----------------------------------------------------------------------===//
16 #include "LiveRangeCalc.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/DenseSet.h"
19 #include "llvm/ADT/None.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/CodeGen/LiveInterval.h"
25 #include "llvm/CodeGen/LiveIntervals.h"
26 #include "llvm/CodeGen/LiveRangeEdit.h"
27 #include "llvm/CodeGen/MachineBasicBlock.h"
28 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
29 #include "llvm/CodeGen/MachineDominators.h"
30 #include "llvm/CodeGen/MachineFunction.h"
31 #include "llvm/CodeGen/MachineInstr.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineLoopInfo.h"
34 #include "llvm/CodeGen/MachineOperand.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/CodeGen/SlotIndexes.h"
37 #include "llvm/CodeGen/TargetInstrInfo.h"
38 #include "llvm/CodeGen/TargetOpcodes.h"
39 #include "llvm/CodeGen/TargetRegisterInfo.h"
40 #include "llvm/CodeGen/TargetSubtargetInfo.h"
41 #include "llvm/CodeGen/VirtRegMap.h"
42 #include "llvm/IR/DebugLoc.h"
43 #include "llvm/MC/LaneBitmask.h"
44 #include "llvm/Support/Allocator.h"
45 #include "llvm/Support/BlockFrequency.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/Debug.h"
48 #include "llvm/Support/ErrorHandling.h"
49 #include "llvm/Support/raw_ostream.h"
59 #define DEBUG_TYPE "regalloc"
61 STATISTIC(NumFinished, "Number of splits finished");
62 STATISTIC(NumSimple, "Number of splits that were simple");
63 STATISTIC(NumCopies, "Number of copies inserted for splitting");
64 STATISTIC(NumRemats, "Number of rematerialized defs for splitting");
65 STATISTIC(NumRepairs, "Number of invalid live ranges repaired");
67 //===----------------------------------------------------------------------===//
68 // Last Insert Point Analysis
69 //===----------------------------------------------------------------------===//
71 InsertPointAnalysis::InsertPointAnalysis(const LiveIntervals &lis,
73 : LIS(lis), LastInsertPoint(BBNum) {}
76 InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI,
77 const MachineBasicBlock &MBB) {
78 unsigned Num = MBB.getNumber();
79 std::pair<SlotIndex, SlotIndex> &LIP = LastInsertPoint[Num];
80 SlotIndex MBBEnd = LIS.getMBBEndIdx(&MBB);
82 SmallVector<const MachineBasicBlock *, 1> EHPadSuccessors;
83 for (const MachineBasicBlock *SMBB : MBB.successors())
85 EHPadSuccessors.push_back(SMBB);
87 // Compute insert points on the first call. The pair is independent of the
88 // current live interval.
89 if (!LIP.first.isValid()) {
90 MachineBasicBlock::const_iterator FirstTerm = MBB.getFirstTerminator();
91 if (FirstTerm == MBB.end())
94 LIP.first = LIS.getInstructionIndex(*FirstTerm);
96 // If there is a landing pad successor, also find the call instruction.
97 if (EHPadSuccessors.empty())
99 // There may not be a call instruction (?) in which case we ignore LPad.
100 LIP.second = LIP.first;
101 for (MachineBasicBlock::const_iterator I = MBB.end(), E = MBB.begin();
105 LIP.second = LIS.getInstructionIndex(*I);
111 // If CurLI is live into a landing pad successor, move the last insert point
112 // back to the call that may throw.
116 if (none_of(EHPadSuccessors, [&](const MachineBasicBlock *EHPad) {
117 return LIS.isLiveInToMBB(CurLI, EHPad);
121 // Find the value leaving MBB.
122 const VNInfo *VNI = CurLI.getVNInfoBefore(MBBEnd);
126 // If the value leaving MBB was defined after the call in MBB, it can't
127 // really be live-in to the landing pad. This can happen if the landing pad
128 // has a PHI, and this register is undef on the exceptional edge.
129 // <rdar://problem/10664933>
130 if (!SlotIndex::isEarlierInstr(VNI->def, LIP.second) && VNI->def < MBBEnd)
133 // Value is properly live-in to the landing pad.
134 // Only allow inserts before the call.
138 MachineBasicBlock::iterator
139 InsertPointAnalysis::getLastInsertPointIter(const LiveInterval &CurLI,
140 MachineBasicBlock &MBB) {
141 SlotIndex LIP = getLastInsertPoint(CurLI, MBB);
142 if (LIP == LIS.getMBBEndIdx(&MBB))
144 return LIS.getInstructionFromIndex(LIP);
147 //===----------------------------------------------------------------------===//
149 //===----------------------------------------------------------------------===//
151 SplitAnalysis::SplitAnalysis(const VirtRegMap &vrm, const LiveIntervals &lis,
152 const MachineLoopInfo &mli)
153 : MF(vrm.getMachineFunction()), VRM(vrm), LIS(lis), Loops(mli),
154 TII(*MF.getSubtarget().getInstrInfo()), IPA(lis, MF.getNumBlockIDs()) {}
156 void SplitAnalysis::clear() {
159 ThroughBlocks.clear();
161 DidRepairRange = false;
164 /// analyzeUses - Count instructions, basic blocks, and loops using CurLI.
165 void SplitAnalysis::analyzeUses() {
166 assert(UseSlots.empty() && "Call clear first");
168 // First get all the defs from the interval values. This provides the correct
169 // slots for early clobbers.
170 for (const VNInfo *VNI : CurLI->valnos)
171 if (!VNI->isPHIDef() && !VNI->isUnused())
172 UseSlots.push_back(VNI->def);
174 // Get use slots form the use-def chain.
175 const MachineRegisterInfo &MRI = MF.getRegInfo();
176 for (MachineOperand &MO : MRI.use_nodbg_operands(CurLI->reg))
178 UseSlots.push_back(LIS.getInstructionIndex(*MO.getParent()).getRegSlot());
180 array_pod_sort(UseSlots.begin(), UseSlots.end());
182 // Remove duplicates, keeping the smaller slot for each instruction.
183 // That is what we want for early clobbers.
184 UseSlots.erase(std::unique(UseSlots.begin(), UseSlots.end(),
185 SlotIndex::isSameInstr),
188 // Compute per-live block info.
189 if (!calcLiveBlockInfo()) {
190 // FIXME: calcLiveBlockInfo found inconsistencies in the live range.
191 // I am looking at you, RegisterCoalescer!
192 DidRepairRange = true;
194 DEBUG(dbgs() << "*** Fixing inconsistent live interval! ***\n");
195 const_cast<LiveIntervals&>(LIS)
196 .shrinkToUses(const_cast<LiveInterval*>(CurLI));
198 ThroughBlocks.clear();
199 bool fixed = calcLiveBlockInfo();
201 assert(fixed && "Couldn't fix broken live interval");
204 DEBUG(dbgs() << "Analyze counted "
205 << UseSlots.size() << " instrs in "
206 << UseBlocks.size() << " blocks, through "
207 << NumThroughBlocks << " blocks.\n");
210 /// calcLiveBlockInfo - Fill the LiveBlocks array with information about blocks
211 /// where CurLI is live.
212 bool SplitAnalysis::calcLiveBlockInfo() {
213 ThroughBlocks.resize(MF.getNumBlockIDs());
214 NumThroughBlocks = NumGapBlocks = 0;
218 LiveInterval::const_iterator LVI = CurLI->begin();
219 LiveInterval::const_iterator LVE = CurLI->end();
221 SmallVectorImpl<SlotIndex>::const_iterator UseI, UseE;
222 UseI = UseSlots.begin();
223 UseE = UseSlots.end();
225 // Loop over basic blocks where CurLI is live.
226 MachineFunction::iterator MFI =
227 LIS.getMBBFromIndex(LVI->start)->getIterator();
231 SlotIndex Start, Stop;
232 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
234 // If the block contains no uses, the range must be live through. At one
235 // point, RegisterCoalescer could create dangling ranges that ended
237 if (UseI == UseE || *UseI >= Stop) {
239 ThroughBlocks.set(BI.MBB->getNumber());
240 // The range shouldn't end mid-block if there are no uses. This shouldn't
245 // This block has uses. Find the first and last uses in the block.
246 BI.FirstInstr = *UseI;
247 assert(BI.FirstInstr >= Start);
249 while (UseI != UseE && *UseI < Stop);
250 BI.LastInstr = UseI[-1];
251 assert(BI.LastInstr < Stop);
253 // LVI is the first live segment overlapping MBB.
254 BI.LiveIn = LVI->start <= Start;
256 // When not live in, the first use should be a def.
258 assert(LVI->start == LVI->valno->def && "Dangling Segment start");
259 assert(LVI->start == BI.FirstInstr && "First instr should be a def");
260 BI.FirstDef = BI.FirstInstr;
263 // Look for gaps in the live range.
265 while (LVI->end < Stop) {
266 SlotIndex LastStop = LVI->end;
267 if (++LVI == LVE || LVI->start >= Stop) {
269 BI.LastInstr = LastStop;
273 if (LastStop < LVI->start) {
274 // There is a gap in the live range. Create duplicate entries for the
275 // live-in snippet and the live-out snippet.
278 // Push the Live-in part.
280 UseBlocks.push_back(BI);
281 UseBlocks.back().LastInstr = LastStop;
283 // Set up BI for the live-out part.
286 BI.FirstInstr = BI.FirstDef = LVI->start;
289 // A Segment that starts in the middle of the block must be a def.
290 assert(LVI->start == LVI->valno->def && "Dangling Segment start");
292 BI.FirstDef = LVI->start;
295 UseBlocks.push_back(BI);
297 // LVI is now at LVE or LVI->end >= Stop.
302 // Live segment ends exactly at Stop. Move to the next segment.
303 if (LVI->end == Stop && ++LVI == LVE)
306 // Pick the next basic block.
307 if (LVI->start < Stop)
310 MFI = LIS.getMBBFromIndex(LVI->start)->getIterator();
313 assert(getNumLiveBlocks() == countLiveBlocks(CurLI) && "Bad block count");
317 unsigned SplitAnalysis::countLiveBlocks(const LiveInterval *cli) const {
320 LiveInterval *li = const_cast<LiveInterval*>(cli);
321 LiveInterval::iterator LVI = li->begin();
322 LiveInterval::iterator LVE = li->end();
325 // Loop over basic blocks where li is live.
326 MachineFunction::const_iterator MFI =
327 LIS.getMBBFromIndex(LVI->start)->getIterator();
328 SlotIndex Stop = LIS.getMBBEndIdx(&*MFI);
331 LVI = li->advanceTo(LVI, Stop);
336 Stop = LIS.getMBBEndIdx(&*MFI);
337 } while (Stop <= LVI->start);
341 bool SplitAnalysis::isOriginalEndpoint(SlotIndex Idx) const {
342 unsigned OrigReg = VRM.getOriginal(CurLI->reg);
343 const LiveInterval &Orig = LIS.getInterval(OrigReg);
344 assert(!Orig.empty() && "Splitting empty interval?");
345 LiveInterval::const_iterator I = Orig.find(Idx);
347 // Range containing Idx should begin at Idx.
348 if (I != Orig.end() && I->start <= Idx)
349 return I->start == Idx;
351 // Range does not contain Idx, previous must end at Idx.
352 return I != Orig.begin() && (--I)->end == Idx;
355 void SplitAnalysis::analyze(const LiveInterval *li) {
361 //===----------------------------------------------------------------------===//
363 //===----------------------------------------------------------------------===//
365 /// Create a new SplitEditor for editing the LiveInterval analyzed by SA.
366 SplitEditor::SplitEditor(SplitAnalysis &sa, AliasAnalysis &aa,
367 LiveIntervals &lis, VirtRegMap &vrm,
368 MachineDominatorTree &mdt,
369 MachineBlockFrequencyInfo &mbfi)
370 : SA(sa), AA(aa), LIS(lis), VRM(vrm),
371 MRI(vrm.getMachineFunction().getRegInfo()), MDT(mdt),
372 TII(*vrm.getMachineFunction().getSubtarget().getInstrInfo()),
373 TRI(*vrm.getMachineFunction().getSubtarget().getRegisterInfo()),
374 MBFI(mbfi), RegAssign(Allocator) {}
376 void SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) {
383 // Reset the LiveRangeCalc instances needed for this spill mode.
384 LRCalc[0].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
385 &LIS.getVNInfoAllocator());
387 LRCalc[1].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
388 &LIS.getVNInfoAllocator());
390 // We don't need an AliasAnalysis since we will only be performing
391 // cheap-as-a-copy remats anyway.
392 Edit->anyRematerializable(nullptr);
395 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
396 LLVM_DUMP_METHOD void SplitEditor::dump() const {
397 if (RegAssign.empty()) {
398 dbgs() << " empty\n";
402 for (RegAssignMap::const_iterator I = RegAssign.begin(); I.valid(); ++I)
403 dbgs() << " [" << I.start() << ';' << I.stop() << "):" << I.value();
408 LiveInterval::SubRange &SplitEditor::getSubRangeForMask(LaneBitmask LM,
410 for (LiveInterval::SubRange &S : LI.subranges())
411 if (S.LaneMask == LM)
413 llvm_unreachable("SubRange for this mask not found");
416 void SplitEditor::addDeadDef(LiveInterval &LI, VNInfo *VNI, bool Original) {
417 if (!LI.hasSubRanges()) {
418 LI.createDeadDef(VNI);
422 SlotIndex Def = VNI->def;
424 // If we are transferring a def from the original interval, make sure
425 // to only update the subranges for which the original subranges had
426 // a def at this location.
427 for (LiveInterval::SubRange &S : LI.subranges()) {
428 auto &PS = getSubRangeForMask(S.LaneMask, Edit->getParent());
429 VNInfo *PV = PS.getVNInfoAt(Def);
430 if (PV != nullptr && PV->def == Def)
431 S.createDeadDef(Def, LIS.getVNInfoAllocator());
434 // This is a new def: either from rematerialization, or from an inserted
435 // copy. Since rematerialization can regenerate a definition of a sub-
436 // register, we need to check which subranges need to be updated.
437 const MachineInstr *DefMI = LIS.getInstructionFromIndex(Def);
438 assert(DefMI != nullptr);
440 for (const MachineOperand &DefOp : DefMI->defs()) {
441 unsigned R = DefOp.getReg();
444 if (unsigned SR = DefOp.getSubReg())
445 LM |= TRI.getSubRegIndexLaneMask(SR);
447 LM = MRI.getMaxLaneMaskForVReg(R);
451 for (LiveInterval::SubRange &S : LI.subranges())
452 if ((S.LaneMask & LM).any())
453 S.createDeadDef(Def, LIS.getVNInfoAllocator());
457 VNInfo *SplitEditor::defValue(unsigned RegIdx,
458 const VNInfo *ParentVNI,
461 assert(ParentVNI && "Mapping NULL value");
462 assert(Idx.isValid() && "Invalid SlotIndex");
463 assert(Edit->getParent().getVNInfoAt(Idx) == ParentVNI && "Bad Parent VNI");
464 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
466 // Create a new value.
467 VNInfo *VNI = LI->getNextValue(Idx, LIS.getVNInfoAllocator());
469 bool Force = LI->hasSubRanges();
470 ValueForcePair FP(Force ? nullptr : VNI, Force);
471 // Use insert for lookup, so we can add missing values with a second lookup.
472 std::pair<ValueMap::iterator, bool> InsP =
473 Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), FP));
475 // This was the first time (RegIdx, ParentVNI) was mapped, and it is not
476 // forced. Keep it as a simple def without any liveness.
477 if (!Force && InsP.second)
480 // If the previous value was a simple mapping, add liveness for it now.
481 if (VNInfo *OldVNI = InsP.first->second.getPointer()) {
482 addDeadDef(*LI, OldVNI, Original);
484 // No longer a simple mapping. Switch to a complex mapping. If the
485 // interval has subranges, make it a forced mapping.
486 InsP.first->second = ValueForcePair(nullptr, Force);
489 // This is a complex mapping, add liveness for VNI
490 addDeadDef(*LI, VNI, Original);
494 void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo &ParentVNI) {
495 ValueForcePair &VFP = Values[std::make_pair(RegIdx, ParentVNI.id)];
496 VNInfo *VNI = VFP.getPointer();
498 // ParentVNI was either unmapped or already complex mapped. Either way, just
499 // set the force bit.
505 // This was previously a single mapping. Make sure the old def is represented
506 // by a trivial live range.
507 addDeadDef(LIS.getInterval(Edit->get(RegIdx)), VNI, false);
509 // Mark as complex mapped, forced.
510 VFP = ValueForcePair(nullptr, true);
513 SlotIndex SplitEditor::buildSingleSubRegCopy(unsigned FromReg, unsigned ToReg,
514 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
515 unsigned SubIdx, LiveInterval &DestLI, bool Late, SlotIndex Def) {
516 const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY);
517 bool FirstCopy = !Def.isValid();
518 MachineInstr *CopyMI = BuildMI(MBB, InsertBefore, DebugLoc(), Desc)
519 .addReg(ToReg, RegState::Define | getUndefRegState(FirstCopy)
520 | getInternalReadRegState(!FirstCopy), SubIdx)
521 .addReg(FromReg, 0, SubIdx);
523 BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator();
525 SlotIndexes &Indexes = *LIS.getSlotIndexes();
526 Def = Indexes.insertMachineInstrInMaps(*CopyMI, Late).getRegSlot();
528 CopyMI->bundleWithPred();
530 LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubIdx);
531 DestLI.refineSubRanges(Allocator, LaneMask,
532 [Def, &Allocator](LiveInterval::SubRange& SR) {
533 SR.createDeadDef(Def, Allocator);
538 SlotIndex SplitEditor::buildCopy(unsigned FromReg, unsigned ToReg,
539 LaneBitmask LaneMask, MachineBasicBlock &MBB,
540 MachineBasicBlock::iterator InsertBefore, bool Late, unsigned RegIdx) {
541 const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY);
542 if (LaneMask.all() || LaneMask == MRI.getMaxLaneMaskForVReg(FromReg)) {
543 // The full vreg is copied.
544 MachineInstr *CopyMI =
545 BuildMI(MBB, InsertBefore, DebugLoc(), Desc, ToReg).addReg(FromReg);
546 SlotIndexes &Indexes = *LIS.getSlotIndexes();
547 return Indexes.insertMachineInstrInMaps(*CopyMI, Late).getRegSlot();
550 // Only a subset of lanes needs to be copied. The following is a simple
551 // heuristic to construct a sequence of COPYs. We could add a target
552 // specific callback if this turns out to be suboptimal.
553 LiveInterval &DestLI = LIS.getInterval(Edit->get(RegIdx));
555 // First pass: Try to find a perfectly matching subregister index. If none
556 // exists find the one covering the most lanemask bits.
557 SmallVector<unsigned, 8> PossibleIndexes;
558 unsigned BestIdx = 0;
559 unsigned BestCover = 0;
560 const TargetRegisterClass *RC = MRI.getRegClass(FromReg);
561 assert(RC == MRI.getRegClass(ToReg) && "Should have same reg class");
562 for (unsigned Idx = 1, E = TRI.getNumSubRegIndices(); Idx < E; ++Idx) {
563 // Is this index even compatible with the given class?
564 if (TRI.getSubClassWithSubReg(RC, Idx) != RC)
566 LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(Idx);
567 // Early exit if we found a perfect match.
568 if (SubRegMask == LaneMask) {
573 // The index must not cover any lanes outside \p LaneMask.
574 if ((SubRegMask & ~LaneMask).any())
577 unsigned PopCount = SubRegMask.getNumLanes();
578 PossibleIndexes.push_back(Idx);
579 if (PopCount > BestCover) {
580 BestCover = PopCount;
585 // Abort if we cannot possibly implement the COPY with the given indexes.
587 report_fatal_error("Impossible to implement partial COPY");
589 SlotIndex Def = buildSingleSubRegCopy(FromReg, ToReg, MBB, InsertBefore,
590 BestIdx, DestLI, Late, SlotIndex());
592 // Greedy heuristic: Keep iterating keeping the best covering subreg index
594 LaneBitmask LanesLeft = LaneMask & ~(TRI.getSubRegIndexLaneMask(BestIdx));
595 while (LanesLeft.any()) {
596 unsigned BestIdx = 0;
597 int BestCover = std::numeric_limits<int>::min();
598 for (unsigned Idx : PossibleIndexes) {
599 LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(Idx);
600 // Early exit if we found a perfect match.
601 if (SubRegMask == LanesLeft) {
606 // Try to cover as much of the remaining lanes as possible but
607 // as few of the already covered lanes as possible.
608 int Cover = (SubRegMask & LanesLeft).getNumLanes()
609 - (SubRegMask & ~LanesLeft).getNumLanes();
610 if (Cover > BestCover) {
617 report_fatal_error("Impossible to implement partial COPY");
619 buildSingleSubRegCopy(FromReg, ToReg, MBB, InsertBefore, BestIdx,
621 LanesLeft &= ~TRI.getSubRegIndexLaneMask(BestIdx);
627 VNInfo *SplitEditor::defFromParent(unsigned RegIdx,
630 MachineBasicBlock &MBB,
631 MachineBasicBlock::iterator I) {
633 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
635 // We may be trying to avoid interference that ends at a deleted instruction,
636 // so always begin RegIdx 0 early and all others late.
637 bool Late = RegIdx != 0;
639 // Attempt cheap-as-a-copy rematerialization.
640 unsigned Original = VRM.getOriginal(Edit->get(RegIdx));
641 LiveInterval &OrigLI = LIS.getInterval(Original);
642 VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);
644 unsigned Reg = LI->reg;
645 bool DidRemat = false;
647 LiveRangeEdit::Remat RM(ParentVNI);
648 RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
649 if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
650 Def = Edit->rematerializeAt(MBB, I, Reg, RM, TRI, Late);
656 LaneBitmask LaneMask;
657 if (LI->hasSubRanges()) {
658 LaneMask = LaneBitmask::getNone();
659 for (LiveInterval::SubRange &S : LI->subranges())
660 LaneMask |= S.LaneMask;
662 LaneMask = LaneBitmask::getAll();
666 Def = buildCopy(Edit->getReg(), Reg, LaneMask, MBB, I, Late, RegIdx);
669 // Define the value in Reg.
670 return defValue(RegIdx, ParentVNI, Def, false);
673 /// Create a new virtual register and live interval.
674 unsigned SplitEditor::openIntv() {
675 // Create the complement as index 0.
677 Edit->createEmptyInterval();
679 // Create the open interval.
680 OpenIdx = Edit->size();
681 Edit->createEmptyInterval();
685 void SplitEditor::selectIntv(unsigned Idx) {
686 assert(Idx != 0 && "Cannot select the complement interval");
687 assert(Idx < Edit->size() && "Can only select previously opened interval");
688 DEBUG(dbgs() << " selectIntv " << OpenIdx << " -> " << Idx << '\n');
692 SlotIndex SplitEditor::enterIntvBefore(SlotIndex Idx) {
693 assert(OpenIdx && "openIntv not called before enterIntvBefore");
694 DEBUG(dbgs() << " enterIntvBefore " << Idx);
695 Idx = Idx.getBaseIndex();
696 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
698 DEBUG(dbgs() << ": not live\n");
701 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
702 MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
703 assert(MI && "enterIntvBefore called with invalid index");
705 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), MI);
709 SlotIndex SplitEditor::enterIntvAfter(SlotIndex Idx) {
710 assert(OpenIdx && "openIntv not called before enterIntvAfter");
711 DEBUG(dbgs() << " enterIntvAfter " << Idx);
712 Idx = Idx.getBoundaryIndex();
713 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
715 DEBUG(dbgs() << ": not live\n");
718 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
719 MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
720 assert(MI && "enterIntvAfter called with invalid index");
722 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(),
723 std::next(MachineBasicBlock::iterator(MI)));
727 SlotIndex SplitEditor::enterIntvAtEnd(MachineBasicBlock &MBB) {
728 assert(OpenIdx && "openIntv not called before enterIntvAtEnd");
729 SlotIndex End = LIS.getMBBEndIdx(&MBB);
730 SlotIndex Last = End.getPrevSlot();
731 DEBUG(dbgs() << " enterIntvAtEnd " << printMBBReference(MBB) << ", "
733 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Last);
735 DEBUG(dbgs() << ": not live\n");
738 DEBUG(dbgs() << ": valno " << ParentVNI->id);
739 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Last, MBB,
740 SA.getLastSplitPointIter(&MBB));
741 RegAssign.insert(VNI->def, End, OpenIdx);
746 /// useIntv - indicate that all instructions in MBB should use OpenLI.
747 void SplitEditor::useIntv(const MachineBasicBlock &MBB) {
748 useIntv(LIS.getMBBStartIdx(&MBB), LIS.getMBBEndIdx(&MBB));
751 void SplitEditor::useIntv(SlotIndex Start, SlotIndex End) {
752 assert(OpenIdx && "openIntv not called before useIntv");
753 DEBUG(dbgs() << " useIntv [" << Start << ';' << End << "):");
754 RegAssign.insert(Start, End, OpenIdx);
758 SlotIndex SplitEditor::leaveIntvAfter(SlotIndex Idx) {
759 assert(OpenIdx && "openIntv not called before leaveIntvAfter");
760 DEBUG(dbgs() << " leaveIntvAfter " << Idx);
762 // The interval must be live beyond the instruction at Idx.
763 SlotIndex Boundary = Idx.getBoundaryIndex();
764 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Boundary);
766 DEBUG(dbgs() << ": not live\n");
767 return Boundary.getNextSlot();
769 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
770 MachineInstr *MI = LIS.getInstructionFromIndex(Boundary);
771 assert(MI && "No instruction at index");
773 // In spill mode, make live ranges as short as possible by inserting the copy
774 // before MI. This is only possible if that instruction doesn't redefine the
775 // value. The inserted COPY is not a kill, and we don't need to recompute
776 // the source live range. The spiller also won't try to hoist this copy.
777 if (SpillMode && !SlotIndex::isSameInstr(ParentVNI->def, Idx) &&
778 MI->readsVirtualRegister(Edit->getReg())) {
779 forceRecompute(0, *ParentVNI);
780 defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI);
784 VNInfo *VNI = defFromParent(0, ParentVNI, Boundary, *MI->getParent(),
785 std::next(MachineBasicBlock::iterator(MI)));
789 SlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) {
790 assert(OpenIdx && "openIntv not called before leaveIntvBefore");
791 DEBUG(dbgs() << " leaveIntvBefore " << Idx);
793 // The interval must be live into the instruction at Idx.
794 Idx = Idx.getBaseIndex();
795 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
797 DEBUG(dbgs() << ": not live\n");
798 return Idx.getNextSlot();
800 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
802 MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
803 assert(MI && "No instruction at index");
804 VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI);
808 SlotIndex SplitEditor::leaveIntvAtTop(MachineBasicBlock &MBB) {
809 assert(OpenIdx && "openIntv not called before leaveIntvAtTop");
810 SlotIndex Start = LIS.getMBBStartIdx(&MBB);
811 DEBUG(dbgs() << " leaveIntvAtTop " << printMBBReference(MBB) << ", "
814 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start);
816 DEBUG(dbgs() << ": not live\n");
820 VNInfo *VNI = defFromParent(0, ParentVNI, Start, MBB,
821 MBB.SkipPHIsLabelsAndDebug(MBB.begin()));
822 RegAssign.insert(Start, VNI->def, OpenIdx);
827 void SplitEditor::overlapIntv(SlotIndex Start, SlotIndex End) {
828 assert(OpenIdx && "openIntv not called before overlapIntv");
829 const VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start);
830 assert(ParentVNI == Edit->getParent().getVNInfoBefore(End) &&
831 "Parent changes value in extended range");
832 assert(LIS.getMBBFromIndex(Start) == LIS.getMBBFromIndex(End) &&
833 "Range cannot span basic blocks");
835 // The complement interval will be extended as needed by LRCalc.extend().
837 forceRecompute(0, *ParentVNI);
838 DEBUG(dbgs() << " overlapIntv [" << Start << ';' << End << "):");
839 RegAssign.insert(Start, End, OpenIdx);
843 //===----------------------------------------------------------------------===//
845 //===----------------------------------------------------------------------===//
847 void SplitEditor::removeBackCopies(SmallVectorImpl<VNInfo*> &Copies) {
848 LiveInterval *LI = &LIS.getInterval(Edit->get(0));
849 DEBUG(dbgs() << "Removing " << Copies.size() << " back-copies.\n");
850 RegAssignMap::iterator AssignI;
851 AssignI.setMap(RegAssign);
853 for (unsigned i = 0, e = Copies.size(); i != e; ++i) {
854 SlotIndex Def = Copies[i]->def;
855 MachineInstr *MI = LIS.getInstructionFromIndex(Def);
856 assert(MI && "No instruction for back-copy");
858 MachineBasicBlock *MBB = MI->getParent();
859 MachineBasicBlock::iterator MBBI(MI);
861 do AtBegin = MBBI == MBB->begin();
862 while (!AtBegin && (--MBBI)->isDebugValue());
864 DEBUG(dbgs() << "Removing " << Def << '\t' << *MI);
865 LIS.removeVRegDefAt(*LI, Def);
866 LIS.RemoveMachineInstrFromMaps(*MI);
867 MI->eraseFromParent();
869 // Adjust RegAssign if a register assignment is killed at Def. We want to
870 // avoid calculating the live range of the source register if possible.
871 AssignI.find(Def.getPrevSlot());
872 if (!AssignI.valid() || AssignI.start() >= Def)
874 // If MI doesn't kill the assigned register, just leave it.
875 if (AssignI.stop() != Def)
877 unsigned RegIdx = AssignI.value();
878 if (AtBegin || !MBBI->readsVirtualRegister(Edit->getReg())) {
879 DEBUG(dbgs() << " cannot find simple kill of RegIdx " << RegIdx << '\n');
880 forceRecompute(RegIdx, *Edit->getParent().getVNInfoAt(Def));
882 SlotIndex Kill = LIS.getInstructionIndex(*MBBI).getRegSlot();
883 DEBUG(dbgs() << " move kill to " << Kill << '\t' << *MBBI);
884 AssignI.setStop(Kill);
890 SplitEditor::findShallowDominator(MachineBasicBlock *MBB,
891 MachineBasicBlock *DefMBB) {
894 assert(MDT.dominates(DefMBB, MBB) && "MBB must be dominated by the def.");
896 const MachineLoopInfo &Loops = SA.Loops;
897 const MachineLoop *DefLoop = Loops.getLoopFor(DefMBB);
898 MachineDomTreeNode *DefDomNode = MDT[DefMBB];
900 // Best candidate so far.
901 MachineBasicBlock *BestMBB = MBB;
902 unsigned BestDepth = std::numeric_limits<unsigned>::max();
905 const MachineLoop *Loop = Loops.getLoopFor(MBB);
907 // MBB isn't in a loop, it doesn't get any better. All dominators have a
908 // higher frequency by definition.
910 DEBUG(dbgs() << "Def in " << printMBBReference(*DefMBB) << " dominates "
911 << printMBBReference(*MBB) << " at depth 0\n");
915 // We'll never be able to exit the DefLoop.
916 if (Loop == DefLoop) {
917 DEBUG(dbgs() << "Def in " << printMBBReference(*DefMBB) << " dominates "
918 << printMBBReference(*MBB) << " in the same loop\n");
922 // Least busy dominator seen so far.
923 unsigned Depth = Loop->getLoopDepth();
924 if (Depth < BestDepth) {
927 DEBUG(dbgs() << "Def in " << printMBBReference(*DefMBB) << " dominates "
928 << printMBBReference(*MBB) << " at depth " << Depth << '\n');
931 // Leave loop by going to the immediate dominator of the loop header.
932 // This is a bigger stride than simply walking up the dominator tree.
933 MachineDomTreeNode *IDom = MDT[Loop->getHeader()]->getIDom();
935 // Too far up the dominator tree?
936 if (!IDom || !MDT.dominates(DefDomNode, IDom))
939 MBB = IDom->getBlock();
943 void SplitEditor::computeRedundantBackCopies(
944 DenseSet<unsigned> &NotToHoistSet, SmallVectorImpl<VNInfo *> &BackCopies) {
945 LiveInterval *LI = &LIS.getInterval(Edit->get(0));
946 LiveInterval *Parent = &Edit->getParent();
947 SmallVector<SmallPtrSet<VNInfo *, 8>, 8> EqualVNs(Parent->getNumValNums());
948 SmallPtrSet<VNInfo *, 8> DominatedVNIs;
950 // Aggregate VNIs having the same value as ParentVNI.
951 for (VNInfo *VNI : LI->valnos) {
954 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
955 EqualVNs[ParentVNI->id].insert(VNI);
958 // For VNI aggregation of each ParentVNI, collect dominated, i.e.,
959 // redundant VNIs to BackCopies.
960 for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) {
961 VNInfo *ParentVNI = Parent->getValNumInfo(i);
962 if (!NotToHoistSet.count(ParentVNI->id))
964 SmallPtrSetIterator<VNInfo *> It1 = EqualVNs[ParentVNI->id].begin();
965 SmallPtrSetIterator<VNInfo *> It2 = It1;
966 for (; It1 != EqualVNs[ParentVNI->id].end(); ++It1) {
968 for (++It2; It2 != EqualVNs[ParentVNI->id].end(); ++It2) {
969 if (DominatedVNIs.count(*It1) || DominatedVNIs.count(*It2))
972 MachineBasicBlock *MBB1 = LIS.getMBBFromIndex((*It1)->def);
973 MachineBasicBlock *MBB2 = LIS.getMBBFromIndex((*It2)->def);
975 DominatedVNIs.insert((*It1)->def < (*It2)->def ? (*It2) : (*It1));
976 } else if (MDT.dominates(MBB1, MBB2)) {
977 DominatedVNIs.insert(*It2);
978 } else if (MDT.dominates(MBB2, MBB1)) {
979 DominatedVNIs.insert(*It1);
983 if (!DominatedVNIs.empty()) {
984 forceRecompute(0, *ParentVNI);
985 for (auto VNI : DominatedVNIs) {
986 BackCopies.push_back(VNI);
988 DominatedVNIs.clear();
993 /// For SM_Size mode, find a common dominator for all the back-copies for
994 /// the same ParentVNI and hoist the backcopies to the dominator BB.
995 /// For SM_Speed mode, if the common dominator is hot and it is not beneficial
996 /// to do the hoisting, simply remove the dominated backcopies for the same
998 void SplitEditor::hoistCopies() {
999 // Get the complement interval, always RegIdx 0.
1000 LiveInterval *LI = &LIS.getInterval(Edit->get(0));
1001 LiveInterval *Parent = &Edit->getParent();
1003 // Track the nearest common dominator for all back-copies for each ParentVNI,
1004 // indexed by ParentVNI->id.
1005 using DomPair = std::pair<MachineBasicBlock *, SlotIndex>;
1006 SmallVector<DomPair, 8> NearestDom(Parent->getNumValNums());
1007 // The total cost of all the back-copies for each ParentVNI.
1008 SmallVector<BlockFrequency, 8> Costs(Parent->getNumValNums());
1009 // The ParentVNI->id set for which hoisting back-copies are not beneficial
1011 DenseSet<unsigned> NotToHoistSet;
1013 // Find the nearest common dominator for parent values with multiple
1014 // back-copies. If a single back-copy dominates, put it in DomPair.second.
1015 for (VNInfo *VNI : LI->valnos) {
1016 if (VNI->isUnused())
1018 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
1019 assert(ParentVNI && "Parent not live at complement def");
1021 // Don't hoist remats. The complement is probably going to disappear
1022 // completely anyway.
1023 if (Edit->didRematerialize(ParentVNI))
1026 MachineBasicBlock *ValMBB = LIS.getMBBFromIndex(VNI->def);
1028 DomPair &Dom = NearestDom[ParentVNI->id];
1030 // Keep directly defined parent values. This is either a PHI or an
1031 // instruction in the complement range. All other copies of ParentVNI
1032 // should be eliminated.
1033 if (VNI->def == ParentVNI->def) {
1034 DEBUG(dbgs() << "Direct complement def at " << VNI->def << '\n');
1035 Dom = DomPair(ValMBB, VNI->def);
1038 // Skip the singly mapped values. There is nothing to gain from hoisting a
1039 // single back-copy.
1040 if (Values.lookup(std::make_pair(0, ParentVNI->id)).getPointer()) {
1041 DEBUG(dbgs() << "Single complement def at " << VNI->def << '\n');
1046 // First time we see ParentVNI. VNI dominates itself.
1047 Dom = DomPair(ValMBB, VNI->def);
1048 } else if (Dom.first == ValMBB) {
1049 // Two defs in the same block. Pick the earlier def.
1050 if (!Dom.second.isValid() || VNI->def < Dom.second)
1051 Dom.second = VNI->def;
1053 // Different basic blocks. Check if one dominates.
1054 MachineBasicBlock *Near =
1055 MDT.findNearestCommonDominator(Dom.first, ValMBB);
1057 // Def ValMBB dominates.
1058 Dom = DomPair(ValMBB, VNI->def);
1059 else if (Near != Dom.first)
1060 // None dominate. Hoist to common dominator, need new def.
1061 Dom = DomPair(Near, SlotIndex());
1062 Costs[ParentVNI->id] += MBFI.getBlockFreq(ValMBB);
1065 DEBUG(dbgs() << "Multi-mapped complement " << VNI->id << '@' << VNI->def
1066 << " for parent " << ParentVNI->id << '@' << ParentVNI->def
1067 << " hoist to " << printMBBReference(*Dom.first) << ' '
1068 << Dom.second << '\n');
1071 // Insert the hoisted copies.
1072 for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) {
1073 DomPair &Dom = NearestDom[i];
1074 if (!Dom.first || Dom.second.isValid())
1076 // This value needs a hoisted copy inserted at the end of Dom.first.
1077 VNInfo *ParentVNI = Parent->getValNumInfo(i);
1078 MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(ParentVNI->def);
1079 // Get a less loopy dominator than Dom.first.
1080 Dom.first = findShallowDominator(Dom.first, DefMBB);
1081 if (SpillMode == SM_Speed &&
1082 MBFI.getBlockFreq(Dom.first) > Costs[ParentVNI->id]) {
1083 NotToHoistSet.insert(ParentVNI->id);
1086 SlotIndex Last = LIS.getMBBEndIdx(Dom.first).getPrevSlot();
1088 defFromParent(0, ParentVNI, Last, *Dom.first,
1089 SA.getLastSplitPointIter(Dom.first))->def;
1092 // Remove redundant back-copies that are now known to be dominated by another
1093 // def with the same value.
1094 SmallVector<VNInfo*, 8> BackCopies;
1095 for (VNInfo *VNI : LI->valnos) {
1096 if (VNI->isUnused())
1098 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def);
1099 const DomPair &Dom = NearestDom[ParentVNI->id];
1100 if (!Dom.first || Dom.second == VNI->def ||
1101 NotToHoistSet.count(ParentVNI->id))
1103 BackCopies.push_back(VNI);
1104 forceRecompute(0, *ParentVNI);
1107 // If it is not beneficial to hoist all the BackCopies, simply remove
1108 // redundant BackCopies in speed mode.
1109 if (SpillMode == SM_Speed && !NotToHoistSet.empty())
1110 computeRedundantBackCopies(NotToHoistSet, BackCopies);
1112 removeBackCopies(BackCopies);
1115 /// transferValues - Transfer all possible values to the new live ranges.
1116 /// Values that were rematerialized are left alone, they need LRCalc.extend().
1117 bool SplitEditor::transferValues() {
1118 bool Skipped = false;
1119 RegAssignMap::const_iterator AssignI = RegAssign.begin();
1120 for (const LiveRange::Segment &S : Edit->getParent()) {
1121 DEBUG(dbgs() << " blit " << S << ':');
1122 VNInfo *ParentVNI = S.valno;
1123 // RegAssign has holes where RegIdx 0 should be used.
1124 SlotIndex Start = S.start;
1125 AssignI.advanceTo(Start);
1128 SlotIndex End = S.end;
1129 if (!AssignI.valid()) {
1131 } else if (AssignI.start() <= Start) {
1132 RegIdx = AssignI.value();
1133 if (AssignI.stop() < End) {
1134 End = AssignI.stop();
1139 End = std::min(End, AssignI.start());
1142 // The interval [Start;End) is continuously mapped to RegIdx, ParentVNI.
1143 DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx
1144 << '(' << printReg(Edit->get(RegIdx)) << ')');
1145 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1147 // Check for a simply defined value that can be blitted directly.
1148 ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id));
1149 if (VNInfo *VNI = VFP.getPointer()) {
1150 DEBUG(dbgs() << ':' << VNI->id);
1151 LI.addSegment(LiveInterval::Segment(Start, End, VNI));
1156 // Skip values with forced recomputation.
1158 DEBUG(dbgs() << "(recalc)");
1164 LiveRangeCalc &LRC = getLRCalc(RegIdx);
1166 // This value has multiple defs in RegIdx, but it wasn't rematerialized,
1167 // so the live range is accurate. Add live-in blocks in [Start;End) to the
1169 MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start)->getIterator();
1170 SlotIndex BlockStart, BlockEnd;
1171 std::tie(BlockStart, BlockEnd) = LIS.getSlotIndexes()->getMBBRange(&*MBB);
1173 // The first block may be live-in, or it may have its own def.
1174 if (Start != BlockStart) {
1175 VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
1176 assert(VNI && "Missing def for complex mapped value");
1177 DEBUG(dbgs() << ':' << VNI->id << "*" << printMBBReference(*MBB));
1178 // MBB has its own def. Is it also live-out?
1179 if (BlockEnd <= End)
1180 LRC.setLiveOutValue(&*MBB, VNI);
1182 // Skip to the next block for live-in.
1184 BlockStart = BlockEnd;
1187 // Handle the live-in blocks covered by [Start;End).
1188 assert(Start <= BlockStart && "Expected live-in block");
1189 while (BlockStart < End) {
1190 DEBUG(dbgs() << ">" << printMBBReference(*MBB));
1191 BlockEnd = LIS.getMBBEndIdx(&*MBB);
1192 if (BlockStart == ParentVNI->def) {
1193 // This block has the def of a parent PHI, so it isn't live-in.
1194 assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?");
1195 VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
1196 assert(VNI && "Missing def for complex mapped parent PHI");
1197 if (End >= BlockEnd)
1198 LRC.setLiveOutValue(&*MBB, VNI); // Live-out as well.
1200 // This block needs a live-in value. The last block covered may not
1203 LRC.addLiveInBlock(LI, MDT[&*MBB], End);
1205 // Live-through, and we don't know the value.
1206 LRC.addLiveInBlock(LI, MDT[&*MBB]);
1207 LRC.setLiveOutValue(&*MBB, nullptr);
1210 BlockStart = BlockEnd;
1214 } while (Start != S.end);
1215 DEBUG(dbgs() << '\n');
1218 LRCalc[0].calculateValues();
1220 LRCalc[1].calculateValues();
1225 static bool removeDeadSegment(SlotIndex Def, LiveRange &LR) {
1226 const LiveRange::Segment *Seg = LR.getSegmentContaining(Def);
1229 if (Seg->end != Def.getDeadSlot())
1231 // This is a dead PHI. Remove it.
1232 LR.removeSegment(*Seg, true);
1236 void SplitEditor::extendPHIRange(MachineBasicBlock &B, LiveRangeCalc &LRC,
1237 LiveRange &LR, LaneBitmask LM,
1238 ArrayRef<SlotIndex> Undefs) {
1239 for (MachineBasicBlock *P : B.predecessors()) {
1240 SlotIndex End = LIS.getMBBEndIdx(P);
1241 SlotIndex LastUse = End.getPrevSlot();
1242 // The predecessor may not have a live-out value. That is OK, like an
1243 // undef PHI operand.
1244 LiveInterval &PLI = Edit->getParent();
1245 // Need the cast because the inputs to ?: would otherwise be deemed
1246 // "incompatible": SubRange vs LiveInterval.
1247 LiveRange &PSR = !LM.all() ? getSubRangeForMask(LM, PLI)
1248 : static_cast<LiveRange&>(PLI);
1249 if (PSR.liveAt(LastUse))
1250 LRC.extend(LR, End, /*PhysReg=*/0, Undefs);
1254 void SplitEditor::extendPHIKillRanges() {
1255 // Extend live ranges to be live-out for successor PHI values.
1257 // Visit each PHI def slot in the parent live interval. If the def is dead,
1258 // remove it. Otherwise, extend the live interval to reach the end indexes
1259 // of all predecessor blocks.
1261 LiveInterval &ParentLI = Edit->getParent();
1262 for (const VNInfo *V : ParentLI.valnos) {
1263 if (V->isUnused() || !V->isPHIDef())
1266 unsigned RegIdx = RegAssign.lookup(V->def);
1267 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1268 LiveRangeCalc &LRC = getLRCalc(RegIdx);
1269 MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
1270 if (!removeDeadSegment(V->def, LI))
1271 extendPHIRange(B, LRC, LI, LaneBitmask::getAll(), /*Undefs=*/{});
1274 SmallVector<SlotIndex, 4> Undefs;
1275 LiveRangeCalc SubLRC;
1277 for (LiveInterval::SubRange &PS : ParentLI.subranges()) {
1278 for (const VNInfo *V : PS.valnos) {
1279 if (V->isUnused() || !V->isPHIDef())
1281 unsigned RegIdx = RegAssign.lookup(V->def);
1282 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1283 LiveInterval::SubRange &S = getSubRangeForMask(PS.LaneMask, LI);
1284 if (removeDeadSegment(V->def, S))
1287 MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
1288 SubLRC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
1289 &LIS.getVNInfoAllocator());
1291 LI.computeSubRangeUndefs(Undefs, PS.LaneMask, MRI, *LIS.getSlotIndexes());
1292 extendPHIRange(B, SubLRC, S, PS.LaneMask, Undefs);
1297 /// rewriteAssigned - Rewrite all uses of Edit->getReg().
1298 void SplitEditor::rewriteAssigned(bool ExtendRanges) {
1300 ExtPoint(const MachineOperand &O, unsigned R, SlotIndex N)
1301 : MO(O), RegIdx(R), Next(N) {}
1308 SmallVector<ExtPoint,4> ExtPoints;
1310 for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()),
1311 RE = MRI.reg_end(); RI != RE;) {
1312 MachineOperand &MO = *RI;
1313 MachineInstr *MI = MO.getParent();
1315 // LiveDebugVariables should have handled all DBG_VALUE instructions.
1316 if (MI->isDebugValue()) {
1317 DEBUG(dbgs() << "Zapping " << *MI);
1322 // <undef> operands don't really read the register, so it doesn't matter
1323 // which register we choose. When the use operand is tied to a def, we must
1324 // use the same register as the def, so just do that always.
1325 SlotIndex Idx = LIS.getInstructionIndex(*MI);
1326 if (MO.isDef() || MO.isUndef())
1327 Idx = Idx.getRegSlot(MO.isEarlyClobber());
1329 // Rewrite to the mapped register at Idx.
1330 unsigned RegIdx = RegAssign.lookup(Idx);
1331 LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
1333 DEBUG(dbgs() << " rewr " << printMBBReference(*MI->getParent()) << '\t'
1334 << Idx << ':' << RegIdx << '\t' << *MI);
1336 // Extend liveness to Idx if the instruction reads reg.
1337 if (!ExtendRanges || MO.isUndef())
1340 // Skip instructions that don't read Reg.
1342 if (!MO.getSubReg() && !MO.isEarlyClobber())
1344 // We may want to extend a live range for a partial redef, or for a use
1345 // tied to an early clobber.
1346 Idx = Idx.getPrevSlot();
1347 if (!Edit->getParent().liveAt(Idx))
1350 Idx = Idx.getRegSlot(true);
1352 SlotIndex Next = Idx.getNextSlot();
1353 if (LI.hasSubRanges()) {
1354 // We have to delay extending subranges until we have seen all operands
1355 // defining the register. This is because a <def,read-undef> operand
1356 // will create an "undef" point, and we cannot extend any subranges
1357 // until all of them have been accounted for.
1359 ExtPoints.push_back(ExtPoint(MO, RegIdx, Next));
1361 LiveRangeCalc &LRC = getLRCalc(RegIdx);
1362 LRC.extend(LI, Next, 0, ArrayRef<SlotIndex>());
1366 for (ExtPoint &EP : ExtPoints) {
1367 LiveInterval &LI = LIS.getInterval(Edit->get(EP.RegIdx));
1368 assert(LI.hasSubRanges());
1370 LiveRangeCalc SubLRC;
1371 unsigned Reg = EP.MO.getReg(), Sub = EP.MO.getSubReg();
1372 LaneBitmask LM = Sub != 0 ? TRI.getSubRegIndexLaneMask(Sub)
1373 : MRI.getMaxLaneMaskForVReg(Reg);
1374 for (LiveInterval::SubRange &S : LI.subranges()) {
1375 if ((S.LaneMask & LM).none())
1377 // The problem here can be that the new register may have been created
1378 // for a partially defined original register. For example:
1379 // %0:subreg_hireg<def,read-undef> = ...
1384 SubLRC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
1385 &LIS.getVNInfoAllocator());
1386 SmallVector<SlotIndex, 4> Undefs;
1387 LI.computeSubRangeUndefs(Undefs, S.LaneMask, MRI, *LIS.getSlotIndexes());
1388 SubLRC.extend(S, EP.Next, 0, Undefs);
1392 for (unsigned R : *Edit) {
1393 LiveInterval &LI = LIS.getInterval(R);
1394 if (!LI.hasSubRanges())
1397 LI.removeEmptySubRanges();
1398 LIS.constructMainRangeFromSubranges(LI);
1402 void SplitEditor::deleteRematVictims() {
1403 SmallVector<MachineInstr*, 8> Dead;
1404 for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I){
1405 LiveInterval *LI = &LIS.getInterval(*I);
1406 for (const LiveRange::Segment &S : LI->segments) {
1407 // Dead defs end at the dead slot.
1408 if (S.end != S.valno->def.getDeadSlot())
1410 if (S.valno->isPHIDef())
1412 MachineInstr *MI = LIS.getInstructionFromIndex(S.valno->def);
1413 assert(MI && "Missing instruction for dead def");
1414 MI->addRegisterDead(LI->reg, &TRI);
1416 if (!MI->allDefsAreDead())
1419 DEBUG(dbgs() << "All defs dead: " << *MI);
1427 Edit->eliminateDeadDefs(Dead, None, &AA);
1430 void SplitEditor::forceRecomputeVNI(const VNInfo &ParentVNI) {
1431 // Fast-path for common case.
1432 if (!ParentVNI.isPHIDef()) {
1433 for (unsigned I = 0, E = Edit->size(); I != E; ++I)
1434 forceRecompute(I, ParentVNI);
1438 // Trace value through phis.
1439 SmallPtrSet<const VNInfo *, 8> Visited; ///< whether VNI was/is in worklist.
1440 SmallVector<const VNInfo *, 4> WorkList;
1441 Visited.insert(&ParentVNI);
1442 WorkList.push_back(&ParentVNI);
1444 const LiveInterval &ParentLI = Edit->getParent();
1445 const SlotIndexes &Indexes = *LIS.getSlotIndexes();
1447 const VNInfo &VNI = *WorkList.back();
1448 WorkList.pop_back();
1449 for (unsigned I = 0, E = Edit->size(); I != E; ++I)
1450 forceRecompute(I, VNI);
1451 if (!VNI.isPHIDef())
1454 MachineBasicBlock &MBB = *Indexes.getMBBFromIndex(VNI.def);
1455 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
1456 SlotIndex PredEnd = Indexes.getMBBEndIdx(Pred);
1457 VNInfo *PredVNI = ParentLI.getVNInfoBefore(PredEnd);
1458 assert(PredVNI && "Value available in PhiVNI predecessor");
1459 if (Visited.insert(PredVNI).second)
1460 WorkList.push_back(PredVNI);
1462 } while(!WorkList.empty());
1465 void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
1468 // At this point, the live intervals in Edit contain VNInfos corresponding to
1469 // the inserted copies.
1471 // Add the original defs from the parent interval.
1472 for (const VNInfo *ParentVNI : Edit->getParent().valnos) {
1473 if (ParentVNI->isUnused())
1475 unsigned RegIdx = RegAssign.lookup(ParentVNI->def);
1476 defValue(RegIdx, ParentVNI, ParentVNI->def, true);
1478 // Force rematted values to be recomputed everywhere.
1479 // The new live ranges may be truncated.
1480 if (Edit->didRematerialize(ParentVNI))
1481 forceRecomputeVNI(*ParentVNI);
1484 // Hoist back-copies to the complement interval when in spill mode.
1485 switch (SpillMode) {
1487 // Leave all back-copies as is.
1491 // hoistCopies will behave differently between size and speed.
1495 // Transfer the simply mapped values, check if any are skipped.
1496 bool Skipped = transferValues();
1498 // Rewrite virtual registers, possibly extending ranges.
1499 rewriteAssigned(Skipped);
1502 extendPHIKillRanges();
1506 // Delete defs that were rematted everywhere.
1508 deleteRematVictims();
1510 // Get rid of unused values and set phi-kill flags.
1511 for (unsigned Reg : *Edit) {
1512 LiveInterval &LI = LIS.getInterval(Reg);
1513 LI.removeEmptySubRanges();
1514 LI.RenumberValues();
1517 // Provide a reverse mapping from original indices to Edit ranges.
1520 for (unsigned i = 0, e = Edit->size(); i != e; ++i)
1521 LRMap->push_back(i);
1524 // Now check if any registers were separated into multiple components.
1525 ConnectedVNInfoEqClasses ConEQ(LIS);
1526 for (unsigned i = 0, e = Edit->size(); i != e; ++i) {
1527 // Don't use iterators, they are invalidated by create() below.
1528 unsigned VReg = Edit->get(i);
1529 LiveInterval &LI = LIS.getInterval(VReg);
1530 SmallVector<LiveInterval*, 8> SplitLIs;
1531 LIS.splitSeparateComponents(LI, SplitLIs);
1532 unsigned Original = VRM.getOriginal(VReg);
1533 for (LiveInterval *SplitLI : SplitLIs)
1534 VRM.setIsSplitFromReg(SplitLI->reg, Original);
1536 // The new intervals all map back to i.
1538 LRMap->resize(Edit->size(), i);
1541 // Calculate spill weight and allocation hints for new intervals.
1542 Edit->calculateRegClassAndHint(VRM.getMachineFunction(), SA.Loops, MBFI);
1544 assert(!LRMap || LRMap->size() == Edit->size());
1547 //===----------------------------------------------------------------------===//
1548 // Single Block Splitting
1549 //===----------------------------------------------------------------------===//
1551 bool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI,
1552 bool SingleInstrs) const {
1553 // Always split for multiple instructions.
1554 if (!BI.isOneInstr())
1556 // Don't split for single instructions unless explicitly requested.
1559 // Splitting a live-through range always makes progress.
1560 if (BI.LiveIn && BI.LiveOut)
1562 // No point in isolating a copy. It has no register class constraints.
1563 if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike())
1565 // Finally, don't isolate an end point that was created by earlier splits.
1566 return isOriginalEndpoint(BI.FirstInstr);
1569 void SplitEditor::splitSingleBlock(const SplitAnalysis::BlockInfo &BI) {
1571 SlotIndex LastSplitPoint = SA.getLastSplitPoint(BI.MBB->getNumber());
1572 SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstInstr,
1574 if (!BI.LiveOut || BI.LastInstr < LastSplitPoint) {
1575 useIntv(SegStart, leaveIntvAfter(BI.LastInstr));
1577 // The last use is after the last valid split point.
1578 SlotIndex SegStop = leaveIntvBefore(LastSplitPoint);
1579 useIntv(SegStart, SegStop);
1580 overlapIntv(SegStop, BI.LastInstr);
1584 //===----------------------------------------------------------------------===//
1585 // Global Live Range Splitting Support
1586 //===----------------------------------------------------------------------===//
1588 // These methods support a method of global live range splitting that uses a
1589 // global algorithm to decide intervals for CFG edges. They will insert split
1590 // points and color intervals in basic blocks while avoiding interference.
1592 // Note that splitSingleBlock is also useful for blocks where both CFG edges
1593 // are on the stack.
1595 void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
1596 unsigned IntvIn, SlotIndex LeaveBefore,
1597 unsigned IntvOut, SlotIndex EnterAfter){
1598 SlotIndex Start, Stop;
1599 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(MBBNum);
1601 DEBUG(dbgs() << "%bb." << MBBNum << " [" << Start << ';' << Stop << ") intf "
1602 << LeaveBefore << '-' << EnterAfter << ", live-through "
1603 << IntvIn << " -> " << IntvOut);
1605 assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks");
1607 assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block");
1608 assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf");
1609 assert((!EnterAfter || EnterAfter >= Start) && "Interference before block");
1611 MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum);
1614 DEBUG(dbgs() << ", spill on entry.\n");
1616 // <<<<<<<<< Possible LeaveBefore interference.
1617 // |-----------| Live through.
1618 // -____________ Spill on entry.
1621 SlotIndex Idx = leaveIntvAtTop(*MBB);
1622 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1628 DEBUG(dbgs() << ", reload on exit.\n");
1630 // >>>>>>> Possible EnterAfter interference.
1631 // |-----------| Live through.
1632 // ___________-- Reload on exit.
1634 selectIntv(IntvOut);
1635 SlotIndex Idx = enterIntvAtEnd(*MBB);
1636 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1641 if (IntvIn == IntvOut && !LeaveBefore && !EnterAfter) {
1642 DEBUG(dbgs() << ", straight through.\n");
1644 // |-----------| Live through.
1645 // ------------- Straight through, same intv, no interference.
1647 selectIntv(IntvOut);
1648 useIntv(Start, Stop);
1652 // We cannot legally insert splits after LSP.
1653 SlotIndex LSP = SA.getLastSplitPoint(MBBNum);
1654 assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf");
1656 if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter ||
1657 LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) {
1658 DEBUG(dbgs() << ", switch avoiding interference.\n");
1660 // >>>> <<<< Non-overlapping EnterAfter/LeaveBefore interference.
1661 // |-----------| Live through.
1662 // ------======= Switch intervals between interference.
1664 selectIntv(IntvOut);
1666 if (LeaveBefore && LeaveBefore < LSP) {
1667 Idx = enterIntvBefore(LeaveBefore);
1670 Idx = enterIntvAtEnd(*MBB);
1673 useIntv(Start, Idx);
1674 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1675 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1679 DEBUG(dbgs() << ", create local intv for interference.\n");
1681 // >>><><><><<<< Overlapping EnterAfter/LeaveBefore interference.
1682 // |-----------| Live through.
1683 // ==---------== Switch intervals before/after interference.
1685 assert(LeaveBefore <= EnterAfter && "Missed case");
1687 selectIntv(IntvOut);
1688 SlotIndex Idx = enterIntvAfter(EnterAfter);
1690 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1693 Idx = leaveIntvBefore(LeaveBefore);
1694 useIntv(Start, Idx);
1695 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1698 void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI,
1699 unsigned IntvIn, SlotIndex LeaveBefore) {
1700 SlotIndex Start, Stop;
1701 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
1703 DEBUG(dbgs() << printMBBReference(*BI.MBB) << " [" << Start << ';' << Stop
1704 << "), uses " << BI.FirstInstr << '-' << BI.LastInstr
1705 << ", reg-in " << IntvIn << ", leave before " << LeaveBefore
1706 << (BI.LiveOut ? ", stack-out" : ", killed in block"));
1708 assert(IntvIn && "Must have register in");
1709 assert(BI.LiveIn && "Must be live-in");
1710 assert((!LeaveBefore || LeaveBefore > Start) && "Bad interference");
1712 if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastInstr)) {
1713 DEBUG(dbgs() << " before interference.\n");
1715 // <<< Interference after kill.
1716 // |---o---x | Killed in block.
1717 // ========= Use IntvIn everywhere.
1720 useIntv(Start, BI.LastInstr);
1724 SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber());
1726 if (!LeaveBefore || LeaveBefore > BI.LastInstr.getBoundaryIndex()) {
1728 // <<< Possible interference after last use.
1729 // |---o---o---| Live-out on stack.
1730 // =========____ Leave IntvIn after last use.
1732 // < Interference after last use.
1733 // |---o---o--o| Live-out on stack, late last use.
1734 // ============ Copy to stack after LSP, overlap IntvIn.
1735 // \_____ Stack interval is live-out.
1737 if (BI.LastInstr < LSP) {
1738 DEBUG(dbgs() << ", spill after last use before interference.\n");
1740 SlotIndex Idx = leaveIntvAfter(BI.LastInstr);
1741 useIntv(Start, Idx);
1742 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1744 DEBUG(dbgs() << ", spill before last split point.\n");
1746 SlotIndex Idx = leaveIntvBefore(LSP);
1747 overlapIntv(Idx, BI.LastInstr);
1748 useIntv(Start, Idx);
1749 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
1754 // The interference is overlapping somewhere we wanted to use IntvIn. That
1755 // means we need to create a local interval that can be allocated a
1756 // different register.
1757 unsigned LocalIntv = openIntv();
1759 DEBUG(dbgs() << ", creating local interval " << LocalIntv << ".\n");
1761 if (!BI.LiveOut || BI.LastInstr < LSP) {
1763 // <<<<<<< Interference overlapping uses.
1764 // |---o---o---| Live-out on stack.
1765 // =====----____ Leave IntvIn before interference, then spill.
1767 SlotIndex To = leaveIntvAfter(BI.LastInstr);
1768 SlotIndex From = enterIntvBefore(LeaveBefore);
1771 useIntv(Start, From);
1772 assert((!LeaveBefore || From <= LeaveBefore) && "Interference");
1776 // <<<<<<< Interference overlapping uses.
1777 // |---o---o--o| Live-out on stack, late last use.
1778 // =====------- Copy to stack before LSP, overlap LocalIntv.
1779 // \_____ Stack interval is live-out.
1781 SlotIndex To = leaveIntvBefore(LSP);
1782 overlapIntv(To, BI.LastInstr);
1783 SlotIndex From = enterIntvBefore(std::min(To, LeaveBefore));
1786 useIntv(Start, From);
1787 assert((!LeaveBefore || From <= LeaveBefore) && "Interference");
1790 void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI,
1791 unsigned IntvOut, SlotIndex EnterAfter) {
1792 SlotIndex Start, Stop;
1793 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB);
1795 DEBUG(dbgs() << printMBBReference(*BI.MBB) << " [" << Start << ';' << Stop
1796 << "), uses " << BI.FirstInstr << '-' << BI.LastInstr
1797 << ", reg-out " << IntvOut << ", enter after " << EnterAfter
1798 << (BI.LiveIn ? ", stack-in" : ", defined in block"));
1800 SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber());
1802 assert(IntvOut && "Must have register out");
1803 assert(BI.LiveOut && "Must be live-out");
1804 assert((!EnterAfter || EnterAfter < LSP) && "Bad interference");
1806 if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstInstr)) {
1807 DEBUG(dbgs() << " after interference.\n");
1809 // >>>> Interference before def.
1810 // | o---o---| Defined in block.
1811 // ========= Use IntvOut everywhere.
1813 selectIntv(IntvOut);
1814 useIntv(BI.FirstInstr, Stop);
1818 if (!EnterAfter || EnterAfter < BI.FirstInstr.getBaseIndex()) {
1819 DEBUG(dbgs() << ", reload after interference.\n");
1821 // >>>> Interference before def.
1822 // |---o---o---| Live-through, stack-in.
1823 // ____========= Enter IntvOut before first use.
1825 selectIntv(IntvOut);
1826 SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstInstr));
1828 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1832 // The interference is overlapping somewhere we wanted to use IntvOut. That
1833 // means we need to create a local interval that can be allocated a
1834 // different register.
1835 DEBUG(dbgs() << ", interference overlaps uses.\n");
1837 // >>>>>>> Interference overlapping uses.
1838 // |---o---o---| Live-through, stack-in.
1839 // ____---====== Create local interval for interference range.
1841 selectIntv(IntvOut);
1842 SlotIndex Idx = enterIntvAfter(EnterAfter);
1844 assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
1847 SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstInstr));