1 //===--- RDFGraph.cpp -----------------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Target-independent, SSA-based data flow graph for register data flow (RDF).
14 #include "llvm/ADT/SetVector.h"
15 #include "llvm/CodeGen/MachineBasicBlock.h"
16 #include "llvm/CodeGen/MachineDominanceFrontier.h"
17 #include "llvm/CodeGen/MachineDominators.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/Target/TargetInstrInfo.h"
21 #include "llvm/Target/TargetRegisterInfo.h"
26 // Printing functions. Have them here first, so that the rest of the code
32 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterRef> &P) {
33 auto &TRI = P.G.getTRI();
34 if (P.Obj.Reg > 0 && P.Obj.Reg < TRI.getNumRegs())
35 OS << TRI.getName(P.Obj.Reg);
37 OS << '#' << P.Obj.Reg;
40 if (P.Obj.Sub < TRI.getNumSubRegIndices())
41 OS << TRI.getSubRegIndexName(P.Obj.Sub);
43 OS << '#' << P.Obj.Sub;
49 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeId> &P) {
50 auto NA = P.G.addr<NodeBase*>(P.Obj);
51 uint16_t Attrs = NA.Addr->getAttrs();
52 uint16_t Kind = NodeAttrs::kind(Attrs);
53 uint16_t Flags = NodeAttrs::flags(Attrs);
54 switch (NodeAttrs::type(Attrs)) {
57 case NodeAttrs::Func: OS << 'f'; break;
58 case NodeAttrs::Block: OS << 'b'; break;
59 case NodeAttrs::Stmt: OS << 's'; break;
60 case NodeAttrs::Phi: OS << 'p'; break;
61 default: OS << "c?"; break;
65 if (Flags & NodeAttrs::Preserving)
67 if (Flags & NodeAttrs::Clobbering)
70 case NodeAttrs::Use: OS << 'u'; break;
71 case NodeAttrs::Def: OS << 'd'; break;
72 case NodeAttrs::Block: OS << 'b'; break;
73 default: OS << "r?"; break;
81 if (Flags & NodeAttrs::Shadow)
87 void printRefHeader(raw_ostream &OS, const NodeAddr<RefNode*> RA,
88 const DataFlowGraph &G) {
89 OS << Print<NodeId>(RA.Id, G) << '<'
90 << Print<RegisterRef>(RA.Addr->getRegRef(), G) << '>';
91 if (RA.Addr->getFlags() & NodeAttrs::Fixed)
97 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<DefNode*>> &P) {
98 printRefHeader(OS, P.Obj, P.G);
100 if (NodeId N = P.Obj.Addr->getReachingDef())
101 OS << Print<NodeId>(N, P.G);
103 if (NodeId N = P.Obj.Addr->getReachedDef())
104 OS << Print<NodeId>(N, P.G);
106 if (NodeId N = P.Obj.Addr->getReachedUse())
107 OS << Print<NodeId>(N, P.G);
109 if (NodeId N = P.Obj.Addr->getSibling())
110 OS << Print<NodeId>(N, P.G);
115 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<UseNode*>> &P) {
116 printRefHeader(OS, P.Obj, P.G);
118 if (NodeId N = P.Obj.Addr->getReachingDef())
119 OS << Print<NodeId>(N, P.G);
121 if (NodeId N = P.Obj.Addr->getSibling())
122 OS << Print<NodeId>(N, P.G);
127 raw_ostream &operator<< (raw_ostream &OS,
128 const Print<NodeAddr<PhiUseNode*>> &P) {
129 printRefHeader(OS, P.Obj, P.G);
131 if (NodeId N = P.Obj.Addr->getReachingDef())
132 OS << Print<NodeId>(N, P.G);
134 if (NodeId N = P.Obj.Addr->getPredecessor())
135 OS << Print<NodeId>(N, P.G);
137 if (NodeId N = P.Obj.Addr->getSibling())
138 OS << Print<NodeId>(N, P.G);
143 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<RefNode*>> &P) {
144 switch (P.Obj.Addr->getKind()) {
146 OS << PrintNode<DefNode*>(P.Obj, P.G);
149 if (P.Obj.Addr->getFlags() & NodeAttrs::PhiRef)
150 OS << PrintNode<PhiUseNode*>(P.Obj, P.G);
152 OS << PrintNode<UseNode*>(P.Obj, P.G);
159 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeList> &P) {
160 unsigned N = P.Obj.size();
161 for (auto I : P.Obj) {
162 OS << Print<NodeId>(I.Id, P.G);
170 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeSet> &P) {
171 unsigned N = P.Obj.size();
172 for (auto I : P.Obj) {
173 OS << Print<NodeId>(I, P.G);
181 template <typename T>
183 PrintListV(const NodeList &L, const DataFlowGraph &G) : List(L), G(G) {}
185 const NodeList &List;
186 const DataFlowGraph &G;
189 template <typename T>
190 raw_ostream &operator<< (raw_ostream &OS, const PrintListV<T> &P) {
191 unsigned N = P.List.size();
192 for (NodeAddr<T> A : P.List) {
193 OS << PrintNode<T>(A, P.G);
202 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<PhiNode*>> &P) {
203 OS << Print<NodeId>(P.Obj.Id, P.G) << ": phi ["
204 << PrintListV<RefNode*>(P.Obj.Addr->members(P.G), P.G) << ']';
209 raw_ostream &operator<< (raw_ostream &OS,
210 const Print<NodeAddr<StmtNode*>> &P) {
211 unsigned Opc = P.Obj.Addr->getCode()->getOpcode();
212 OS << Print<NodeId>(P.Obj.Id, P.G) << ": " << P.G.getTII().getName(Opc)
213 << " [" << PrintListV<RefNode*>(P.Obj.Addr->members(P.G), P.G) << ']';
218 raw_ostream &operator<< (raw_ostream &OS,
219 const Print<NodeAddr<InstrNode*>> &P) {
220 switch (P.Obj.Addr->getKind()) {
222 OS << PrintNode<PhiNode*>(P.Obj, P.G);
224 case NodeAttrs::Stmt:
225 OS << PrintNode<StmtNode*>(P.Obj, P.G);
228 OS << "instr? " << Print<NodeId>(P.Obj.Id, P.G);
235 raw_ostream &operator<< (raw_ostream &OS,
236 const Print<NodeAddr<BlockNode*>> &P) {
237 auto *BB = P.Obj.Addr->getCode();
238 unsigned NP = BB->pred_size();
240 auto PrintBBs = [&OS,&P] (std::vector<int> Ns) -> void {
241 unsigned N = Ns.size();
249 OS << Print<NodeId>(P.Obj.Id, P.G) << ": === BB#" << BB->getNumber()
250 << " === preds(" << NP << "): ";
251 for (auto I : BB->predecessors())
252 Ns.push_back(I->getNumber());
255 unsigned NS = BB->succ_size();
256 OS << " succs(" << NS << "): ";
258 for (auto I : BB->successors())
259 Ns.push_back(I->getNumber());
263 for (auto I : P.Obj.Addr->members(P.G))
264 OS << PrintNode<InstrNode*>(I, P.G) << '\n';
269 raw_ostream &operator<< (raw_ostream &OS,
270 const Print<NodeAddr<FuncNode*>> &P) {
271 OS << "DFG dump:[\n" << Print<NodeId>(P.Obj.Id, P.G) << ": Function: "
272 << P.Obj.Addr->getCode()->getName() << '\n';
273 for (auto I : P.Obj.Addr->members(P.G))
274 OS << PrintNode<BlockNode*>(I, P.G) << '\n';
280 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterSet> &P) {
283 OS << ' ' << Print<RegisterRef>(I, P.G);
289 raw_ostream &operator<< (raw_ostream &OS,
290 const Print<DataFlowGraph::DefStack> &P) {
291 for (auto I = P.Obj.top(), E = P.Obj.bottom(); I != E; ) {
292 OS << Print<NodeId>(I->Id, P.G)
293 << '<' << Print<RegisterRef>(I->Addr->getRegRef(), P.G) << '>';
304 // Node allocation functions.
306 // Node allocator is like a slab memory allocator: it allocates blocks of
307 // memory in sizes that are multiples of the size of a node. Each block has
308 // the same size. Nodes are allocated from the currently active block, and
309 // when it becomes full, a new one is created.
310 // There is a mapping scheme between node id and its location in a block,
311 // and within that block is described in the header file.
313 void NodeAllocator::startNewBlock() {
314 void *T = MemPool.Allocate(NodesPerBlock*NodeMemSize, NodeMemSize);
315 char *P = static_cast<char*>(T);
317 // Check if the block index is still within the allowed range, i.e. less
318 // than 2^N, where N is the number of bits in NodeId for the block index.
319 // BitsPerIndex is the number of bits per node index.
320 assert((Blocks.size() < ((size_t)1 << (8*sizeof(NodeId)-BitsPerIndex))) &&
321 "Out of bits for block index");
325 bool NodeAllocator::needNewBlock() {
329 char *ActiveBegin = Blocks.back();
330 uint32_t Index = (ActiveEnd-ActiveBegin)/NodeMemSize;
331 return Index >= NodesPerBlock;
334 NodeAddr<NodeBase*> NodeAllocator::New() {
338 uint32_t ActiveB = Blocks.size()-1;
339 uint32_t Index = (ActiveEnd - Blocks[ActiveB])/NodeMemSize;
340 NodeAddr<NodeBase*> NA = { reinterpret_cast<NodeBase*>(ActiveEnd),
341 makeId(ActiveB, Index) };
342 ActiveEnd += NodeMemSize;
346 NodeId NodeAllocator::id(const NodeBase *P) const {
347 uintptr_t A = reinterpret_cast<uintptr_t>(P);
348 for (unsigned i = 0, n = Blocks.size(); i != n; ++i) {
349 uintptr_t B = reinterpret_cast<uintptr_t>(Blocks[i]);
350 if (A < B || A >= B + NodesPerBlock*NodeMemSize)
352 uint32_t Idx = (A-B)/NodeMemSize;
353 return makeId(i, Idx);
355 llvm_unreachable("Invalid node address");
358 void NodeAllocator::clear() {
365 // Insert node NA after "this" in the circular chain.
366 void NodeBase::append(NodeAddr<NodeBase*> NA) {
368 // If NA is already "next", do nothing.
376 // Fundamental node manipulator functions.
378 // Obtain the register reference from a reference node.
379 RegisterRef RefNode::getRegRef() const {
380 assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
381 if (NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef)
383 assert(Ref.Op != nullptr);
384 return { Ref.Op->getReg(), Ref.Op->getSubReg() };
387 // Set the register reference in the reference node directly (for references
389 void RefNode::setRegRef(RegisterRef RR) {
390 assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
391 assert(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef);
395 // Set the register reference in the reference node based on a machine
396 // operand (for references in statement nodes).
397 void RefNode::setRegRef(MachineOperand *Op) {
398 assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
399 assert(!(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef));
403 // Get the owner of a given reference node.
404 NodeAddr<NodeBase*> RefNode::getOwner(const DataFlowGraph &G) {
405 NodeAddr<NodeBase*> NA = G.addr<NodeBase*>(getNext());
407 while (NA.Addr != this) {
408 if (NA.Addr->getType() == NodeAttrs::Code)
410 NA = G.addr<NodeBase*>(NA.Addr->getNext());
412 llvm_unreachable("No owner in circular list");
415 // Connect the def node to the reaching def node.
416 void DefNode::linkToDef(NodeId Self, NodeAddr<DefNode*> DA) {
418 Ref.Sib = DA.Addr->getReachedDef();
419 DA.Addr->setReachedDef(Self);
422 // Connect the use node to the reaching def node.
423 void UseNode::linkToDef(NodeId Self, NodeAddr<DefNode*> DA) {
425 Ref.Sib = DA.Addr->getReachedUse();
426 DA.Addr->setReachedUse(Self);
429 // Get the first member of the code node.
430 NodeAddr<NodeBase*> CodeNode::getFirstMember(const DataFlowGraph &G) const {
431 if (Code.FirstM == 0)
432 return NodeAddr<NodeBase*>();
433 return G.addr<NodeBase*>(Code.FirstM);
436 // Get the last member of the code node.
437 NodeAddr<NodeBase*> CodeNode::getLastMember(const DataFlowGraph &G) const {
439 return NodeAddr<NodeBase*>();
440 return G.addr<NodeBase*>(Code.LastM);
443 // Add node NA at the end of the member list of the given code node.
444 void CodeNode::addMember(NodeAddr<NodeBase*> NA, const DataFlowGraph &G) {
445 auto ML = getLastMember(G);
450 NodeId Self = G.id(this);
451 NA.Addr->setNext(Self);
456 // Add node NA after member node MA in the given code node.
457 void CodeNode::addMemberAfter(NodeAddr<NodeBase*> MA, NodeAddr<NodeBase*> NA,
458 const DataFlowGraph &G) {
460 if (Code.LastM == MA.Id)
464 // Remove member node NA from the given code node.
465 void CodeNode::removeMember(NodeAddr<NodeBase*> NA, const DataFlowGraph &G) {
466 auto MA = getFirstMember(G);
469 // Special handling if the member to remove is the first member.
470 if (MA.Id == NA.Id) {
471 if (Code.LastM == MA.Id) {
472 // If it is the only member, set both first and last to 0.
473 Code.FirstM = Code.LastM = 0;
475 // Otherwise, advance the first member.
476 Code.FirstM = MA.Addr->getNext();
481 while (MA.Addr != this) {
482 NodeId MX = MA.Addr->getNext();
484 MA.Addr->setNext(NA.Addr->getNext());
485 // If the member to remove happens to be the last one, update the
487 if (Code.LastM == NA.Id)
491 MA = G.addr<NodeBase*>(MX);
493 llvm_unreachable("No such member");
496 // Return the list of all members of the code node.
497 NodeList CodeNode::members(const DataFlowGraph &G) const {
498 static auto True = [] (NodeAddr<NodeBase*>) -> bool { return true; };
499 return members_if(True, G);
502 // Return the owner of the given instr node.
503 NodeAddr<NodeBase*> InstrNode::getOwner(const DataFlowGraph &G) {
504 NodeAddr<NodeBase*> NA = G.addr<NodeBase*>(getNext());
506 while (NA.Addr != this) {
507 assert(NA.Addr->getType() == NodeAttrs::Code);
508 if (NA.Addr->getKind() == NodeAttrs::Block)
510 NA = G.addr<NodeBase*>(NA.Addr->getNext());
512 llvm_unreachable("No owner in circular list");
515 // Add the phi node PA to the given block node.
516 void BlockNode::addPhi(NodeAddr<PhiNode*> PA, const DataFlowGraph &G) {
517 auto M = getFirstMember(G);
523 assert(M.Addr->getType() == NodeAttrs::Code);
524 if (M.Addr->getKind() == NodeAttrs::Stmt) {
525 // If the first member of the block is a statement, insert the phi as
528 PA.Addr->setNext(M.Id);
530 // If the first member is a phi, find the last phi, and append PA to it.
531 assert(M.Addr->getKind() == NodeAttrs::Phi);
532 NodeAddr<NodeBase*> MN = M;
535 MN = G.addr<NodeBase*>(M.Addr->getNext());
536 assert(MN.Addr->getType() == NodeAttrs::Code);
537 } while (MN.Addr->getKind() == NodeAttrs::Phi);
539 // M is the last phi.
540 addMemberAfter(M, PA, G);
544 // Find the block node corresponding to the machine basic block BB in the
546 NodeAddr<BlockNode*> FuncNode::findBlock(const MachineBasicBlock *BB,
547 const DataFlowGraph &G) const {
548 auto EqBB = [BB] (NodeAddr<NodeBase*> NA) -> bool {
549 return NodeAddr<BlockNode*>(NA).Addr->getCode() == BB;
551 NodeList Ms = members_if(EqBB, G);
554 return NodeAddr<BlockNode*>();
557 // Get the block node for the entry block in the given function.
558 NodeAddr<BlockNode*> FuncNode::getEntryBlock(const DataFlowGraph &G) {
559 MachineBasicBlock *EntryB = &getCode()->front();
560 return findBlock(EntryB, G);
564 // Register aliasing information.
566 // In theory, the lane information could be used to determine register
567 // covering (and aliasing), but depending on the sub-register structure,
568 // the lane mask information may be missing. The covering information
569 // must be available for this framework to work, so relying solely on
570 // the lane data is not sufficient.
572 // Determine whether RA covers RB.
573 bool RegisterAliasInfo::covers(RegisterRef RA, RegisterRef RB) const {
576 if (TargetRegisterInfo::isVirtualRegister(RA.Reg)) {
577 assert(TargetRegisterInfo::isVirtualRegister(RB.Reg));
578 if (RA.Reg != RB.Reg)
582 return TRI.composeSubRegIndices(RA.Sub, RB.Sub) == RA.Sub;
585 assert(TargetRegisterInfo::isPhysicalRegister(RA.Reg) &&
586 TargetRegisterInfo::isPhysicalRegister(RB.Reg));
587 unsigned A = RA.Sub != 0 ? TRI.getSubReg(RA.Reg, RA.Sub) : RA.Reg;
588 unsigned B = RB.Sub != 0 ? TRI.getSubReg(RB.Reg, RB.Sub) : RB.Reg;
589 return TRI.isSubRegister(A, B);
592 // Determine whether RR is covered by the set of references RRs.
593 bool RegisterAliasInfo::covers(const RegisterSet &RRs, RegisterRef RR) const {
597 // For virtual registers, we cannot accurately determine covering based
598 // on subregisters. If RR itself is not present in RRs, but it has a sub-
599 // register reference, check for the super-register alone. Otherwise,
600 // assume non-covering.
601 if (TargetRegisterInfo::isVirtualRegister(RR.Reg)) {
603 return RRs.count({RR.Reg, 0});
607 // If any super-register of RR is present, then RR is covered.
608 unsigned Reg = RR.Sub == 0 ? RR.Reg : TRI.getSubReg(RR.Reg, RR.Sub);
609 for (MCSuperRegIterator SR(Reg, &TRI); SR.isValid(); ++SR)
610 if (RRs.count({*SR, 0}))
616 // Get the list of references aliased to RR.
617 std::vector<RegisterRef> RegisterAliasInfo::getAliasSet(RegisterRef RR) const {
618 // Do not include RR in the alias set. For virtual registers return an
620 std::vector<RegisterRef> AS;
621 if (TargetRegisterInfo::isVirtualRegister(RR.Reg))
623 assert(TargetRegisterInfo::isPhysicalRegister(RR.Reg));
626 R = TRI.getSubReg(RR.Reg, RR.Sub);
628 for (MCRegAliasIterator AI(R, &TRI, false); AI.isValid(); ++AI)
629 AS.push_back(RegisterRef({*AI, 0}));
633 // Check whether RA and RB are aliased.
634 bool RegisterAliasInfo::alias(RegisterRef RA, RegisterRef RB) const {
635 bool VirtA = TargetRegisterInfo::isVirtualRegister(RA.Reg);
636 bool VirtB = TargetRegisterInfo::isVirtualRegister(RB.Reg);
637 bool PhysA = TargetRegisterInfo::isPhysicalRegister(RA.Reg);
638 bool PhysB = TargetRegisterInfo::isPhysicalRegister(RB.Reg);
644 if (RA.Reg != RB.Reg)
646 // RA and RB refer to the same register. If any of them refer to the
647 // whole register, they must be aliased.
648 if (RA.Sub == 0 || RB.Sub == 0)
650 unsigned SA = TRI.getSubRegIdxSize(RA.Sub);
651 unsigned OA = TRI.getSubRegIdxOffset(RA.Sub);
652 unsigned SB = TRI.getSubRegIdxSize(RB.Sub);
653 unsigned OB = TRI.getSubRegIdxOffset(RB.Sub);
654 if (OA <= OB && OA+SA > OB)
656 if (OB <= OA && OB+SB > OA)
661 assert(PhysA && PhysB);
662 (void)PhysA, (void)PhysB;
663 unsigned A = RA.Sub ? TRI.getSubReg(RA.Reg, RA.Sub) : RA.Reg;
664 unsigned B = RB.Sub ? TRI.getSubReg(RB.Reg, RB.Sub) : RB.Reg;
665 for (MCRegAliasIterator I(A, &TRI, true); I.isValid(); ++I)
672 // Target operand information.
675 // For a given instruction, check if there are any bits of RR that can remain
676 // unchanged across this def.
677 bool TargetOperandInfo::isPreserving(const MachineInstr &In, unsigned OpNum)
679 return TII.isPredicated(In);
682 // Check if the definition of RR produces an unspecified value.
683 bool TargetOperandInfo::isClobbering(const MachineInstr &In, unsigned OpNum)
686 if (In.getOperand(OpNum).isImplicit())
691 // Check if the given instruction specifically requires
692 bool TargetOperandInfo::isFixedReg(const MachineInstr &In, unsigned OpNum)
694 if (In.isCall() || In.isReturn() || In.isInlineAsm())
696 // Check for a tail call.
698 for (auto &O : In.operands())
699 if (O.isGlobal() || O.isSymbol())
702 const MCInstrDesc &D = In.getDesc();
703 if (!D.getImplicitDefs() && !D.getImplicitUses())
705 const MachineOperand &Op = In.getOperand(OpNum);
706 // If there is a sub-register, treat the operand as non-fixed. Currently,
707 // fixed registers are those that are listed in the descriptor as implicit
708 // uses or defs, and those lists do not allow sub-registers.
709 if (Op.getSubReg() != 0)
711 unsigned Reg = Op.getReg();
712 const MCPhysReg *ImpR = Op.isDef() ? D.getImplicitDefs()
713 : D.getImplicitUses();
724 // The data flow graph construction.
727 DataFlowGraph::DataFlowGraph(MachineFunction &mf, const TargetInstrInfo &tii,
728 const TargetRegisterInfo &tri, const MachineDominatorTree &mdt,
729 const MachineDominanceFrontier &mdf, const RegisterAliasInfo &rai,
730 const TargetOperandInfo &toi)
731 : TimeG("rdf"), MF(mf), TII(tii), TRI(tri), MDT(mdt), MDF(mdf), RAI(rai),
736 // The implementation of the definition stack.
737 // Each register reference has its own definition stack. In particular,
738 // for a register references "Reg" and "Reg:subreg" will each have their
739 // own definition stacks.
741 // Construct a stack iterator.
742 DataFlowGraph::DefStack::Iterator::Iterator(const DataFlowGraph::DefStack &S,
745 // Initialize to bottom.
749 // Initialize to the top, i.e. top-most non-delimiter (or 0, if empty).
750 Pos = DS.Stack.size();
751 while (Pos > 0 && DS.isDelimiter(DS.Stack[Pos-1]))
755 // Return the size of the stack, including block delimiters.
756 unsigned DataFlowGraph::DefStack::size() const {
758 for (auto I = top(), E = bottom(); I != E; I.down())
763 // Remove the top entry from the stack. Remove all intervening delimiters
764 // so that after this, the stack is either empty, or the top of the stack
765 // is a non-delimiter.
766 void DataFlowGraph::DefStack::pop() {
768 unsigned P = nextDown(Stack.size());
772 // Push a delimiter for block node N on the stack.
773 void DataFlowGraph::DefStack::start_block(NodeId N) {
775 Stack.push_back(NodeAddr<DefNode*>(nullptr, N));
778 // Remove all nodes from the top of the stack, until the delimited for
779 // block node N is encountered. Remove the delimiter as well. In effect,
780 // this will remove from the stack all definitions from block N.
781 void DataFlowGraph::DefStack::clear_block(NodeId N) {
783 unsigned P = Stack.size();
785 bool Found = isDelimiter(Stack[P-1], N);
790 // This will also remove the delimiter, if found.
794 // Move the stack iterator up by one.
795 unsigned DataFlowGraph::DefStack::nextUp(unsigned P) const {
796 // Get the next valid position after P (skipping all delimiters).
797 // The input position P does not have to point to a non-delimiter.
798 unsigned SS = Stack.size();
803 IsDelim = isDelimiter(Stack[P-1]);
804 } while (P < SS && IsDelim);
809 // Move the stack iterator down by one.
810 unsigned DataFlowGraph::DefStack::nextDown(unsigned P) const {
811 // Get the preceding valid position before P (skipping all delimiters).
812 // The input position P does not have to point to a non-delimiter.
813 assert(P > 0 && P <= Stack.size());
814 bool IsDelim = isDelimiter(Stack[P-1]);
818 IsDelim = isDelimiter(Stack[P-1]);
819 } while (P > 0 && IsDelim);
824 // Node management functions.
826 // Get the pointer to the node with the id N.
827 NodeBase *DataFlowGraph::ptr(NodeId N) const {
830 return Memory.ptr(N);
833 // Get the id of the node at the address P.
834 NodeId DataFlowGraph::id(const NodeBase *P) const {
840 // Allocate a new node and set the attributes to Attrs.
841 NodeAddr<NodeBase*> DataFlowGraph::newNode(uint16_t Attrs) {
842 NodeAddr<NodeBase*> P = Memory.New();
844 P.Addr->setAttrs(Attrs);
848 // Make a copy of the given node B, except for the data-flow links, which
850 NodeAddr<NodeBase*> DataFlowGraph::cloneNode(const NodeAddr<NodeBase*> B) {
851 NodeAddr<NodeBase*> NA = newNode(0);
852 memcpy(NA.Addr, B.Addr, sizeof(NodeBase));
853 // Ref nodes need to have the data-flow links reset.
854 if (NA.Addr->getType() == NodeAttrs::Ref) {
855 NodeAddr<RefNode*> RA = NA;
856 RA.Addr->setReachingDef(0);
857 RA.Addr->setSibling(0);
858 if (NA.Addr->getKind() == NodeAttrs::Def) {
859 NodeAddr<DefNode*> DA = NA;
860 DA.Addr->setReachedDef(0);
861 DA.Addr->setReachedUse(0);
868 // Allocation routines for specific node types/kinds.
870 NodeAddr<UseNode*> DataFlowGraph::newUse(NodeAddr<InstrNode*> Owner,
871 MachineOperand &Op, uint16_t Flags) {
872 NodeAddr<UseNode*> UA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags);
873 UA.Addr->setRegRef(&Op);
877 NodeAddr<PhiUseNode*> DataFlowGraph::newPhiUse(NodeAddr<PhiNode*> Owner,
878 RegisterRef RR, NodeAddr<BlockNode*> PredB, uint16_t Flags) {
879 NodeAddr<PhiUseNode*> PUA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags);
880 assert(Flags & NodeAttrs::PhiRef);
881 PUA.Addr->setRegRef(RR);
882 PUA.Addr->setPredecessor(PredB.Id);
886 NodeAddr<DefNode*> DataFlowGraph::newDef(NodeAddr<InstrNode*> Owner,
887 MachineOperand &Op, uint16_t Flags) {
888 NodeAddr<DefNode*> DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags);
889 DA.Addr->setRegRef(&Op);
893 NodeAddr<DefNode*> DataFlowGraph::newDef(NodeAddr<InstrNode*> Owner,
894 RegisterRef RR, uint16_t Flags) {
895 NodeAddr<DefNode*> DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags);
896 assert(Flags & NodeAttrs::PhiRef);
897 DA.Addr->setRegRef(RR);
901 NodeAddr<PhiNode*> DataFlowGraph::newPhi(NodeAddr<BlockNode*> Owner) {
902 NodeAddr<PhiNode*> PA = newNode(NodeAttrs::Code | NodeAttrs::Phi);
903 Owner.Addr->addPhi(PA, *this);
907 NodeAddr<StmtNode*> DataFlowGraph::newStmt(NodeAddr<BlockNode*> Owner,
909 NodeAddr<StmtNode*> SA = newNode(NodeAttrs::Code | NodeAttrs::Stmt);
910 SA.Addr->setCode(MI);
911 Owner.Addr->addMember(SA, *this);
915 NodeAddr<BlockNode*> DataFlowGraph::newBlock(NodeAddr<FuncNode*> Owner,
916 MachineBasicBlock *BB) {
917 NodeAddr<BlockNode*> BA = newNode(NodeAttrs::Code | NodeAttrs::Block);
918 BA.Addr->setCode(BB);
919 Owner.Addr->addMember(BA, *this);
923 NodeAddr<FuncNode*> DataFlowGraph::newFunc(MachineFunction *MF) {
924 NodeAddr<FuncNode*> FA = newNode(NodeAttrs::Code | NodeAttrs::Func);
925 FA.Addr->setCode(MF);
929 // Build the data flow graph.
930 void DataFlowGraph::build(unsigned Options) {
938 auto BA = newBlock(Func, &B);
940 if (I.isDebugValue())
946 // Collect information about block references.
947 NodeAddr<BlockNode*> EA = Func.Addr->getEntryBlock(*this);
949 buildBlockRefs(EA, RefM);
951 // Add function-entry phi nodes.
952 MachineRegisterInfo &MRI = MF.getRegInfo();
953 for (auto I = MRI.livein_begin(), E = MRI.livein_end(); I != E; ++I) {
954 NodeAddr<PhiNode*> PA = newPhi(EA);
955 RegisterRef RR = { I->first, 0 };
956 uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
957 NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
958 PA.Addr->addMember(DA, *this);
961 // Build a map "PhiM" which will contain, for each block, the set
962 // of references that will require phi definitions in that block.
964 auto Blocks = Func.Addr->members(*this);
965 for (NodeAddr<BlockNode*> BA : Blocks)
966 recordDefsForDF(PhiM, RefM, BA);
967 for (NodeAddr<BlockNode*> BA : Blocks)
968 buildPhis(PhiM, RefM, BA);
970 // Link all the refs. This will recursively traverse the dominator tree.
972 linkBlockRefs(DM, EA);
974 // Finally, remove all unused phi nodes.
975 if (!(Options & BuildOptions::KeepDeadPhis))
979 // For each stack in the map DefM, push the delimiter for block B on it.
980 void DataFlowGraph::markBlock(NodeId B, DefStackMap &DefM) {
981 // Push block delimiters.
982 for (auto I = DefM.begin(), E = DefM.end(); I != E; ++I)
983 I->second.start_block(B);
986 // Remove all definitions coming from block B from each stack in DefM.
987 void DataFlowGraph::releaseBlock(NodeId B, DefStackMap &DefM) {
988 // Pop all defs from this block from the definition stack. Defs that were
989 // added to the map during the traversal of instructions will not have a
990 // delimiter, but for those, the whole stack will be emptied.
991 for (auto I = DefM.begin(), E = DefM.end(); I != E; ++I)
992 I->second.clear_block(B);
994 // Finally, remove empty stacks from the map.
995 for (auto I = DefM.begin(), E = DefM.end(), NextI = I; I != E; I = NextI) {
996 NextI = std::next(I);
997 // This preserves the validity of iterators other than I.
998 if (I->second.empty())
1003 // Push all definitions from the instruction node IA to an appropriate
1005 void DataFlowGraph::pushDefs(NodeAddr<InstrNode*> IA, DefStackMap &DefM) {
1006 NodeList Defs = IA.Addr->members_if(IsDef, *this);
1009 RegisterSet Defined;
1012 // The important objectives of this function are:
1013 // - to be able to handle instructions both while the graph is being
1014 // constructed, and after the graph has been constructed, and
1015 // - maintain proper ordering of definitions on the stack for each
1016 // register reference:
1017 // - if there are two or more related defs in IA (i.e. coming from
1018 // the same machine operand), then only push one def on the stack,
1019 // - if there are multiple unrelated defs of non-overlapping
1020 // subregisters of S, then the stack for S will have both (in an
1021 // unspecified order), but the order does not matter from the data-
1022 // -flow perspective.
1024 for (NodeAddr<DefNode*> DA : Defs) {
1025 if (Visited.count(DA.Id))
1027 NodeList Rel = getRelatedRefs(IA, DA);
1028 NodeAddr<DefNode*> PDA = Rel.front();
1029 // Push the definition on the stack for the register and all aliases.
1030 RegisterRef RR = PDA.Addr->getRegRef();
1032 // Assert if the register is defined in two or more unrelated defs.
1033 // This could happen if there are two or more def operands defining it.
1034 if (!Defined.insert(RR).second) {
1035 auto *MI = NodeAddr<StmtNode*>(IA).Addr->getCode();
1036 dbgs() << "Multiple definitions of register: "
1037 << Print<RegisterRef>(RR, *this) << " in\n " << *MI
1038 << "in BB#" << MI->getParent()->getNumber() << '\n';
1039 llvm_unreachable(nullptr);
1043 for (auto A : RAI.getAliasSet(RR)) {
1047 // Mark all the related defs as visited.
1049 Visited.insert(T.Id);
1053 // Return the list of all reference nodes related to RA, including RA itself.
1054 // See "getNextRelated" for the meaning of a "related reference".
1055 NodeList DataFlowGraph::getRelatedRefs(NodeAddr<InstrNode*> IA,
1056 NodeAddr<RefNode*> RA) const {
1057 assert(IA.Id != 0 && RA.Id != 0);
1060 NodeId Start = RA.Id;
1063 RA = getNextRelated(IA, RA);
1064 } while (RA.Id != 0 && RA.Id != Start);
1069 // Clear all information in the graph.
1070 void DataFlowGraph::reset() {
1072 Func = NodeAddr<FuncNode*>();
1076 // Return the next reference node in the instruction node IA that is related
1077 // to RA. Conceptually, two reference nodes are related if they refer to the
1078 // same instance of a register access, but differ in flags or other minor
1079 // characteristics. Specific examples of related nodes are shadow reference
1081 // Return the equivalent of nullptr if there are no more related references.
1082 NodeAddr<RefNode*> DataFlowGraph::getNextRelated(NodeAddr<InstrNode*> IA,
1083 NodeAddr<RefNode*> RA) const {
1084 assert(IA.Id != 0 && RA.Id != 0);
1086 auto Related = [RA](NodeAddr<RefNode*> TA) -> bool {
1087 if (TA.Addr->getKind() != RA.Addr->getKind())
1089 if (TA.Addr->getRegRef() != RA.Addr->getRegRef())
1093 auto RelatedStmt = [&Related,RA](NodeAddr<RefNode*> TA) -> bool {
1094 return Related(TA) &&
1095 &RA.Addr->getOp() == &TA.Addr->getOp();
1097 auto RelatedPhi = [&Related,RA](NodeAddr<RefNode*> TA) -> bool {
1100 if (TA.Addr->getKind() != NodeAttrs::Use)
1102 // For phi uses, compare predecessor blocks.
1103 const NodeAddr<const PhiUseNode*> TUA = TA;
1104 const NodeAddr<const PhiUseNode*> RUA = RA;
1105 return TUA.Addr->getPredecessor() == RUA.Addr->getPredecessor();
1108 RegisterRef RR = RA.Addr->getRegRef();
1109 if (IA.Addr->getKind() == NodeAttrs::Stmt)
1110 return RA.Addr->getNextRef(RR, RelatedStmt, true, *this);
1111 return RA.Addr->getNextRef(RR, RelatedPhi, true, *this);
1114 // Find the next node related to RA in IA that satisfies condition P.
1115 // If such a node was found, return a pair where the second element is the
1116 // located node. If such a node does not exist, return a pair where the
1117 // first element is the element after which such a node should be inserted,
1118 // and the second element is a null-address.
1119 template <typename Predicate>
1120 std::pair<NodeAddr<RefNode*>,NodeAddr<RefNode*>>
1121 DataFlowGraph::locateNextRef(NodeAddr<InstrNode*> IA, NodeAddr<RefNode*> RA,
1122 Predicate P) const {
1123 assert(IA.Id != 0 && RA.Id != 0);
1125 NodeAddr<RefNode*> NA;
1126 NodeId Start = RA.Id;
1128 NA = getNextRelated(IA, RA);
1129 if (NA.Id == 0 || NA.Id == Start)
1136 if (NA.Id != 0 && NA.Id != Start)
1137 return std::make_pair(RA, NA);
1138 return std::make_pair(RA, NodeAddr<RefNode*>());
1141 // Get the next shadow node in IA corresponding to RA, and optionally create
1142 // such a node if it does not exist.
1143 NodeAddr<RefNode*> DataFlowGraph::getNextShadow(NodeAddr<InstrNode*> IA,
1144 NodeAddr<RefNode*> RA, bool Create) {
1145 assert(IA.Id != 0 && RA.Id != 0);
1147 uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow;
1148 auto IsShadow = [Flags] (NodeAddr<RefNode*> TA) -> bool {
1149 return TA.Addr->getFlags() == Flags;
1151 auto Loc = locateNextRef(IA, RA, IsShadow);
1152 if (Loc.second.Id != 0 || !Create)
1155 // Create a copy of RA and mark is as shadow.
1156 NodeAddr<RefNode*> NA = cloneNode(RA);
1157 NA.Addr->setFlags(Flags | NodeAttrs::Shadow);
1158 IA.Addr->addMemberAfter(Loc.first, NA, *this);
1162 // Get the next shadow node in IA corresponding to RA. Return null-address
1163 // if such a node does not exist.
1164 NodeAddr<RefNode*> DataFlowGraph::getNextShadow(NodeAddr<InstrNode*> IA,
1165 NodeAddr<RefNode*> RA) const {
1166 assert(IA.Id != 0 && RA.Id != 0);
1167 uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow;
1168 auto IsShadow = [Flags] (NodeAddr<RefNode*> TA) -> bool {
1169 return TA.Addr->getFlags() == Flags;
1171 return locateNextRef(IA, RA, IsShadow).second;
1174 // Create a new statement node in the block node BA that corresponds to
1175 // the machine instruction MI.
1176 void DataFlowGraph::buildStmt(NodeAddr<BlockNode*> BA, MachineInstr &In) {
1177 auto SA = newStmt(BA, &In);
1179 auto isCall = [] (const MachineInstr &In) -> bool {
1184 for (auto &Op : In.operands())
1185 if (Op.isGlobal() || Op.isSymbol())
1190 // Collect a set of registers that this instruction implicitly uses
1191 // or defines. Implicit operands from an instruction will be ignored
1192 // unless they are listed here.
1193 RegisterSet ImpUses, ImpDefs;
1194 if (const uint16_t *ImpD = In.getDesc().getImplicitDefs())
1195 while (uint16_t R = *ImpD++)
1196 ImpDefs.insert({R, 0});
1197 if (const uint16_t *ImpU = In.getDesc().getImplicitUses())
1198 while (uint16_t R = *ImpU++)
1199 ImpUses.insert({R, 0});
1201 bool NeedsImplicit = isCall(In) || In.isInlineAsm() || In.isReturn();
1202 bool IsPredicated = TII.isPredicated(In);
1203 unsigned NumOps = In.getNumOperands();
1205 // Avoid duplicate implicit defs. This will not detect cases of implicit
1206 // defs that define registers that overlap, but it is not clear how to
1207 // interpret that in the absence of explicit defs. Overlapping explicit
1208 // defs are likely illegal already.
1209 RegisterSet DoneDefs;
1210 // Process explicit defs first.
1211 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1212 MachineOperand &Op = In.getOperand(OpN);
1213 if (!Op.isReg() || !Op.isDef() || Op.isImplicit())
1215 RegisterRef RR = { Op.getReg(), Op.getSubReg() };
1216 uint16_t Flags = NodeAttrs::None;
1217 if (TOI.isPreserving(In, OpN))
1218 Flags |= NodeAttrs::Preserving;
1219 if (TOI.isClobbering(In, OpN))
1220 Flags |= NodeAttrs::Clobbering;
1221 if (TOI.isFixedReg(In, OpN))
1222 Flags |= NodeAttrs::Fixed;
1223 NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1224 SA.Addr->addMember(DA, *this);
1225 DoneDefs.insert(RR);
1228 // Process implicit defs, skipping those that have already been added
1230 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1231 MachineOperand &Op = In.getOperand(OpN);
1232 if (!Op.isReg() || !Op.isDef() || !Op.isImplicit())
1234 RegisterRef RR = { Op.getReg(), Op.getSubReg() };
1235 if (!NeedsImplicit && !ImpDefs.count(RR))
1237 if (DoneDefs.count(RR))
1239 uint16_t Flags = NodeAttrs::None;
1240 if (TOI.isPreserving(In, OpN))
1241 Flags |= NodeAttrs::Preserving;
1242 if (TOI.isClobbering(In, OpN))
1243 Flags |= NodeAttrs::Clobbering;
1244 if (TOI.isFixedReg(In, OpN))
1245 Flags |= NodeAttrs::Fixed;
1246 NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1247 SA.Addr->addMember(DA, *this);
1248 DoneDefs.insert(RR);
1251 for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1252 MachineOperand &Op = In.getOperand(OpN);
1253 if (!Op.isReg() || !Op.isUse())
1255 RegisterRef RR = { Op.getReg(), Op.getSubReg() };
1256 // Add implicit uses on return and call instructions, and on predicated
1257 // instructions regardless of whether or not they appear in the instruction
1258 // descriptor's list.
1259 bool Implicit = Op.isImplicit();
1260 bool TakeImplicit = NeedsImplicit || IsPredicated;
1261 if (Implicit && !TakeImplicit && !ImpUses.count(RR))
1263 uint16_t Flags = NodeAttrs::None;
1264 if (TOI.isFixedReg(In, OpN))
1265 Flags |= NodeAttrs::Fixed;
1266 NodeAddr<UseNode*> UA = newUse(SA, Op, Flags);
1267 SA.Addr->addMember(UA, *this);
1271 // Build a map that for each block will have the set of all references from
1272 // that block, and from all blocks dominated by it.
1273 void DataFlowGraph::buildBlockRefs(NodeAddr<BlockNode*> BA,
1274 BlockRefsMap &RefM) {
1275 auto &Refs = RefM[BA.Id];
1276 MachineDomTreeNode *N = MDT.getNode(BA.Addr->getCode());
1279 MachineBasicBlock *SB = I->getBlock();
1280 auto SBA = Func.Addr->findBlock(SB, *this);
1281 buildBlockRefs(SBA, RefM);
1282 const auto &SRs = RefM[SBA.Id];
1283 Refs.insert(SRs.begin(), SRs.end());
1286 for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this))
1287 for (NodeAddr<RefNode*> RA : IA.Addr->members(*this))
1288 Refs.insert(RA.Addr->getRegRef());
1291 // Scan all defs in the block node BA and record in PhiM the locations of
1292 // phi nodes corresponding to these defs.
1293 void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, BlockRefsMap &RefM,
1294 NodeAddr<BlockNode*> BA) {
1295 // Check all defs from block BA and record them in each block in BA's
1296 // iterated dominance frontier. This information will later be used to
1297 // create phi nodes.
1298 MachineBasicBlock *BB = BA.Addr->getCode();
1300 auto DFLoc = MDF.find(BB);
1301 if (DFLoc == MDF.end() || DFLoc->second.empty())
1304 // Traverse all instructions in the block and collect the set of all
1305 // defined references. For each reference there will be a phi created
1306 // in the block's iterated dominance frontier.
1307 // This is done to make sure that each defined reference gets only one
1308 // phi node, even if it is defined multiple times.
1310 for (auto I : BA.Addr->members(*this)) {
1311 assert(I.Addr->getType() == NodeAttrs::Code);
1312 assert(I.Addr->getKind() == NodeAttrs::Phi ||
1313 I.Addr->getKind() == NodeAttrs::Stmt);
1314 NodeAddr<InstrNode*> IA = I;
1315 for (NodeAddr<RefNode*> RA : IA.Addr->members_if(IsDef, *this))
1316 Defs.insert(RA.Addr->getRegRef());
1319 // Finally, add the set of defs to each block in the iterated dominance
1321 const MachineDominanceFrontier::DomSetType &DF = DFLoc->second;
1322 SetVector<MachineBasicBlock*> IDF(DF.begin(), DF.end());
1323 for (unsigned i = 0; i < IDF.size(); ++i) {
1324 auto F = MDF.find(IDF[i]);
1326 IDF.insert(F->second.begin(), F->second.end());
1329 // Get the register references that are reachable from this block.
1330 RegisterSet &Refs = RefM[BA.Id];
1331 for (auto DB : IDF) {
1332 auto DBA = Func.Addr->findBlock(DB, *this);
1333 const auto &Rs = RefM[DBA.Id];
1334 Refs.insert(Rs.begin(), Rs.end());
1337 for (auto DB : IDF) {
1338 auto DBA = Func.Addr->findBlock(DB, *this);
1339 PhiM[DBA.Id].insert(Defs.begin(), Defs.end());
1343 // Given the locations of phi nodes in the map PhiM, create the phi nodes
1344 // that are located in the block node BA.
1345 void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, BlockRefsMap &RefM,
1346 NodeAddr<BlockNode*> BA) {
1347 // Check if this blocks has any DF defs, i.e. if there are any defs
1348 // that this block is in the iterated dominance frontier of.
1349 auto HasDF = PhiM.find(BA.Id);
1350 if (HasDF == PhiM.end() || HasDF->second.empty())
1353 // First, remove all R in Refs in such that there exists T in Refs
1354 // such that T covers R. In other words, only leave those refs that
1355 // are not covered by another ref (i.e. maximal with respect to covering).
1357 auto MaxCoverIn = [this] (RegisterRef RR, RegisterSet &RRs) -> RegisterRef {
1359 if (I != RR && RAI.covers(I, RR))
1365 for (auto I : HasDF->second)
1366 MaxDF.insert(MaxCoverIn(I, HasDF->second));
1368 std::vector<RegisterRef> MaxRefs;
1369 auto &RefB = RefM[BA.Id];
1370 for (auto I : MaxDF)
1371 MaxRefs.push_back(MaxCoverIn(I, RefB));
1373 // Now, for each R in MaxRefs, get the alias closure of R. If the closure
1374 // only has R in it, create a phi a def for R. Otherwise, create a phi,
1375 // and add a def for each S in the closure.
1377 // Sort the refs so that the phis will be created in a deterministic order.
1378 std::sort(MaxRefs.begin(), MaxRefs.end());
1379 // Remove duplicates.
1380 auto NewEnd = std::unique(MaxRefs.begin(), MaxRefs.end());
1381 MaxRefs.erase(NewEnd, MaxRefs.end());
1383 auto Aliased = [this,&MaxRefs](RegisterRef RR,
1384 std::vector<unsigned> &Closure) -> bool {
1385 for (auto I : Closure)
1386 if (RAI.alias(RR, MaxRefs[I]))
1391 // Prepare a list of NodeIds of the block's predecessors.
1392 std::vector<NodeId> PredList;
1393 const MachineBasicBlock *MBB = BA.Addr->getCode();
1394 for (auto PB : MBB->predecessors()) {
1395 auto B = Func.Addr->findBlock(PB, *this);
1396 PredList.push_back(B.Id);
1399 while (!MaxRefs.empty()) {
1400 // Put the first element in the closure, and then add all subsequent
1401 // elements from MaxRefs to it, if they alias at least one element
1402 // already in the closure.
1403 // ClosureIdx: vector of indices in MaxRefs of members of the closure.
1404 std::vector<unsigned> ClosureIdx = { 0 };
1405 for (unsigned i = 1; i != MaxRefs.size(); ++i)
1406 if (Aliased(MaxRefs[i], ClosureIdx))
1407 ClosureIdx.push_back(i);
1409 // Build a phi for the closure.
1410 unsigned CS = ClosureIdx.size();
1411 NodeAddr<PhiNode*> PA = newPhi(BA);
1414 for (unsigned X = 0; X != CS; ++X) {
1415 RegisterRef RR = MaxRefs[ClosureIdx[X]];
1416 uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
1417 NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
1418 PA.Addr->addMember(DA, *this);
1421 for (auto P : PredList) {
1422 auto PBA = addr<BlockNode*>(P);
1423 for (unsigned X = 0; X != CS; ++X) {
1424 RegisterRef RR = MaxRefs[ClosureIdx[X]];
1425 NodeAddr<PhiUseNode*> PUA = newPhiUse(PA, RR, PBA);
1426 PA.Addr->addMember(PUA, *this);
1430 // Erase from MaxRefs all elements in the closure.
1431 auto Begin = MaxRefs.begin();
1432 for (unsigned i = ClosureIdx.size(); i != 0; --i)
1433 MaxRefs.erase(Begin + ClosureIdx[i-1]);
1437 // Remove any unneeded phi nodes that were created during the build process.
1438 void DataFlowGraph::removeUnusedPhis() {
1439 // This will remove unused phis, i.e. phis where each def does not reach
1440 // any uses or other defs. This will not detect or remove circular phi
1441 // chains that are otherwise dead. Unused/dead phis are created during
1442 // the build process and this function is intended to remove these cases
1443 // that are easily determinable to be unnecessary.
1445 SetVector<NodeId> PhiQ;
1446 for (NodeAddr<BlockNode*> BA : Func.Addr->members(*this)) {
1447 for (auto P : BA.Addr->members_if(IsPhi, *this))
1451 static auto HasUsedDef = [](NodeList &Ms) -> bool {
1453 if (M.Addr->getKind() != NodeAttrs::Def)
1455 NodeAddr<DefNode*> DA = M;
1456 if (DA.Addr->getReachedDef() != 0 || DA.Addr->getReachedUse() != 0)
1462 // Any phi, if it is removed, may affect other phis (make them dead).
1463 // For each removed phi, collect the potentially affected phis and add
1464 // them back to the queue.
1465 while (!PhiQ.empty()) {
1466 auto PA = addr<PhiNode*>(PhiQ[0]);
1468 NodeList Refs = PA.Addr->members(*this);
1469 if (HasUsedDef(Refs))
1471 for (NodeAddr<RefNode*> RA : Refs) {
1472 if (NodeId RD = RA.Addr->getReachingDef()) {
1473 auto RDA = addr<DefNode*>(RD);
1474 NodeAddr<InstrNode*> OA = RDA.Addr->getOwner(*this);
1478 if (RA.Addr->isDef())
1479 unlinkDef(RA, true);
1481 unlinkUse(RA, true);
1483 NodeAddr<BlockNode*> BA = PA.Addr->getOwner(*this);
1484 BA.Addr->removeMember(PA, *this);
1488 // For a given reference node TA in an instruction node IA, connect the
1489 // reaching def of TA to the appropriate def node. Create any shadow nodes
1491 template <typename T>
1492 void DataFlowGraph::linkRefUp(NodeAddr<InstrNode*> IA, NodeAddr<T> TA,
1496 RegisterRef RR = TA.Addr->getRegRef();
1499 // References from the def stack that have been examined so far.
1502 for (auto I = DS.top(), E = DS.bottom(); I != E; I.down()) {
1503 RegisterRef QR = I->Addr->getRegRef();
1504 auto AliasQR = [QR,this] (RegisterRef RR) -> bool {
1505 return RAI.alias(QR, RR);
1507 bool PrecUp = RAI.covers(QR, RR);
1508 // Skip all defs that are aliased to any of the defs that we have already
1509 // seen. If we encounter a covering def, stop the stack traversal early.
1510 if (std::any_of(Defs.begin(), Defs.end(), AliasQR)) {
1515 // The reaching def.
1516 NodeAddr<DefNode*> RDA = *I;
1518 // Pick the reached node.
1522 // Mark the existing ref as "shadow" and create a new shadow.
1523 TAP.Addr->setFlags(TAP.Addr->getFlags() | NodeAttrs::Shadow);
1524 TAP = getNextShadow(IA, TAP, true);
1528 TAP.Addr->linkToDef(TAP.Id, RDA);
1536 // Create data-flow links for all reference nodes in the statement node SA.
1537 void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, NodeAddr<StmtNode*> SA) {
1540 // Link all nodes (upwards in the data-flow) with their reaching defs.
1541 for (NodeAddr<RefNode*> RA : SA.Addr->members(*this)) {
1542 uint16_t Kind = RA.Addr->getKind();
1543 assert(Kind == NodeAttrs::Def || Kind == NodeAttrs::Use);
1544 RegisterRef RR = RA.Addr->getRegRef();
1545 // Do not process multiple defs of the same reference.
1546 if (Kind == NodeAttrs::Def && Defs.count(RR))
1550 auto F = DefM.find(RR);
1551 if (F == DefM.end())
1553 DefStack &DS = F->second;
1554 if (Kind == NodeAttrs::Use)
1555 linkRefUp<UseNode*>(SA, RA, DS);
1556 else if (Kind == NodeAttrs::Def)
1557 linkRefUp<DefNode*>(SA, RA, DS);
1559 llvm_unreachable("Unexpected node in instruction");
1563 // Create data-flow links for all instructions in the block node BA. This
1564 // will include updating any phi nodes in BA.
1565 void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, NodeAddr<BlockNode*> BA) {
1566 // Push block delimiters.
1567 markBlock(BA.Id, DefM);
1569 assert(BA.Addr && "block node address is needed to create a data-flow link");
1570 // For each non-phi instruction in the block, link all the defs and uses
1571 // to their reaching defs. For any member of the block (including phis),
1572 // push the defs on the corresponding stacks.
1573 for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this)) {
1574 // Ignore phi nodes here. They will be linked part by part from the
1576 if (IA.Addr->getKind() == NodeAttrs::Stmt)
1577 linkStmtRefs(DefM, IA);
1579 // Push the definitions on the stack.
1583 // Recursively process all children in the dominator tree.
1584 MachineDomTreeNode *N = MDT.getNode(BA.Addr->getCode());
1586 MachineBasicBlock *SB = I->getBlock();
1587 auto SBA = Func.Addr->findBlock(SB, *this);
1588 linkBlockRefs(DefM, SBA);
1591 // Link the phi uses from the successor blocks.
1592 auto IsUseForBA = [BA](NodeAddr<NodeBase*> NA) -> bool {
1593 if (NA.Addr->getKind() != NodeAttrs::Use)
1595 assert(NA.Addr->getFlags() & NodeAttrs::PhiRef);
1596 NodeAddr<PhiUseNode*> PUA = NA;
1597 return PUA.Addr->getPredecessor() == BA.Id;
1599 MachineBasicBlock *MBB = BA.Addr->getCode();
1600 for (auto SB : MBB->successors()) {
1601 auto SBA = Func.Addr->findBlock(SB, *this);
1602 for (NodeAddr<InstrNode*> IA : SBA.Addr->members_if(IsPhi, *this)) {
1603 // Go over each phi use associated with MBB, and link it.
1604 for (auto U : IA.Addr->members_if(IsUseForBA, *this)) {
1605 NodeAddr<PhiUseNode*> PUA = U;
1606 RegisterRef RR = PUA.Addr->getRegRef();
1607 linkRefUp<UseNode*>(IA, PUA, DefM[RR]);
1612 // Pop all defs from this block from the definition stacks.
1613 releaseBlock(BA.Id, DefM);
1616 // Remove the use node UA from any data-flow and structural links.
1617 void DataFlowGraph::unlinkUseDF(NodeAddr<UseNode*> UA) {
1618 NodeId RD = UA.Addr->getReachingDef();
1619 NodeId Sib = UA.Addr->getSibling();
1626 auto RDA = addr<DefNode*>(RD);
1627 auto TA = addr<UseNode*>(RDA.Addr->getReachedUse());
1628 if (TA.Id == UA.Id) {
1629 RDA.Addr->setReachedUse(Sib);
1633 while (TA.Id != 0) {
1634 NodeId S = TA.Addr->getSibling();
1636 TA.Addr->setSibling(UA.Addr->getSibling());
1639 TA = addr<UseNode*>(S);
1643 // Remove the def node DA from any data-flow and structural links.
1644 void DataFlowGraph::unlinkDefDF(NodeAddr<DefNode*> DA) {
1652 // ... -- | DA | -- ... -- 0 : sibling chain of DA
1657 // | ... : Siblings (defs)
1661 // ... : sibling chain of reached uses
1663 NodeId RD = DA.Addr->getReachingDef();
1665 // Visit all siblings of the reached def and reset their reaching defs.
1666 // Also, defs reached by DA are now "promoted" to being reached by RD,
1667 // so all of them will need to be spliced into the sibling chain where
1669 auto getAllNodes = [this] (NodeId N) -> NodeList {
1672 auto RA = addr<RefNode*>(N);
1673 // Keep the nodes in the exact sibling order.
1675 N = RA.Addr->getSibling();
1679 NodeList ReachedDefs = getAllNodes(DA.Addr->getReachedDef());
1680 NodeList ReachedUses = getAllNodes(DA.Addr->getReachedUse());
1683 for (NodeAddr<RefNode*> I : ReachedDefs)
1684 I.Addr->setSibling(0);
1685 for (NodeAddr<RefNode*> I : ReachedUses)
1686 I.Addr->setSibling(0);
1688 for (NodeAddr<DefNode*> I : ReachedDefs)
1689 I.Addr->setReachingDef(RD);
1690 for (NodeAddr<UseNode*> I : ReachedUses)
1691 I.Addr->setReachingDef(RD);
1693 NodeId Sib = DA.Addr->getSibling();
1699 // Update the reaching def node and remove DA from the sibling list.
1700 auto RDA = addr<DefNode*>(RD);
1701 auto TA = addr<DefNode*>(RDA.Addr->getReachedDef());
1702 if (TA.Id == DA.Id) {
1703 // If DA is the first reached def, just update the RD's reached def
1704 // to the DA's sibling.
1705 RDA.Addr->setReachedDef(Sib);
1707 // Otherwise, traverse the sibling list of the reached defs and remove
1709 while (TA.Id != 0) {
1710 NodeId S = TA.Addr->getSibling();
1712 TA.Addr->setSibling(Sib);
1715 TA = addr<DefNode*>(S);
1719 // Splice the DA's reached defs into the RDA's reached def chain.
1720 if (!ReachedDefs.empty()) {
1721 auto Last = NodeAddr<DefNode*>(ReachedDefs.back());
1722 Last.Addr->setSibling(RDA.Addr->getReachedDef());
1723 RDA.Addr->setReachedDef(ReachedDefs.front().Id);
1725 // Splice the DA's reached uses into the RDA's reached use chain.
1726 if (!ReachedUses.empty()) {
1727 auto Last = NodeAddr<UseNode*>(ReachedUses.back());
1728 Last.Addr->setSibling(RDA.Addr->getReachedUse());
1729 RDA.Addr->setReachedUse(ReachedUses.front().Id);