1 //===----- HexagonPacketizer.cpp - vliw packetizer ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This implements a simple VLIW packetizer using DFA. The packetizer works on
11 // machine basic blocks. For each instruction I in BB, the packetizer consults
12 // the DFA to see if machine resources are available to execute I. If so, the
13 // packetizer checks if I depends on any instruction J in the current packet.
14 // If no dependency is found, I is added to current packet and machine resource
15 // is marked as taken. If any dependency is found, a target API call is made to
16 // prune the dependence.
18 //===----------------------------------------------------------------------===//
19 #include "HexagonRegisterInfo.h"
20 #include "HexagonSubtarget.h"
21 #include "HexagonTargetMachine.h"
22 #include "HexagonVLIWPacketizer.h"
23 #include "llvm/Analysis/AliasAnalysis.h"
24 #include "llvm/CodeGen/MachineDominators.h"
25 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
26 #include "llvm/CodeGen/MachineFunctionPass.h"
27 #include "llvm/CodeGen/MachineLoopInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/Debug.h"
35 #define DEBUG_TYPE "packets"
37 static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden,
38 cl::ZeroOrMore, cl::init(false),
39 cl::desc("Disable Hexagon packetizer pass"));
41 static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles",
42 cl::ZeroOrMore, cl::Hidden, cl::init(true),
43 cl::desc("Allow non-solo packetization of volatile memory references"));
45 static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false),
46 cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC"));
48 static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores",
49 cl::init(false), cl::Hidden, cl::ZeroOrMore,
50 cl::desc("Disable vector double new-value-stores"));
52 extern cl::opt<bool> ScheduleInlineAsm;
55 FunctionPass *createHexagonPacketizer();
56 void initializeHexagonPacketizerPass(PassRegistry&);
61 class HexagonPacketizer : public MachineFunctionPass {
64 HexagonPacketizer() : MachineFunctionPass(ID) {
65 initializeHexagonPacketizerPass(*PassRegistry::getPassRegistry());
68 void getAnalysisUsage(AnalysisUsage &AU) const override {
70 AU.addRequired<AAResultsWrapperPass>();
71 AU.addRequired<MachineBranchProbabilityInfo>();
72 AU.addRequired<MachineDominatorTree>();
73 AU.addRequired<MachineLoopInfo>();
74 AU.addPreserved<MachineDominatorTree>();
75 AU.addPreserved<MachineLoopInfo>();
76 MachineFunctionPass::getAnalysisUsage(AU);
78 const char *getPassName() const override {
79 return "Hexagon Packetizer";
81 bool runOnMachineFunction(MachineFunction &Fn) override;
82 MachineFunctionProperties getRequiredProperties() const override {
83 return MachineFunctionProperties().set(
84 MachineFunctionProperties::Property::AllVRegsAllocated);
88 const HexagonInstrInfo *HII;
89 const HexagonRegisterInfo *HRI;
92 char HexagonPacketizer::ID = 0;
95 INITIALIZE_PASS_BEGIN(HexagonPacketizer, "packets", "Hexagon Packetizer",
97 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
98 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
99 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
100 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
101 INITIALIZE_PASS_END(HexagonPacketizer, "packets", "Hexagon Packetizer",
105 HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF,
106 MachineLoopInfo &MLI, AliasAnalysis *AA,
107 const MachineBranchProbabilityInfo *MBPI)
108 : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI) {
109 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
110 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
112 addMutation(make_unique<HexagonSubtarget::HexagonDAGMutation>());
115 // Check if FirstI modifies a register that SecondI reads.
116 static bool hasWriteToReadDep(const MachineInstr &FirstI,
117 const MachineInstr &SecondI,
118 const TargetRegisterInfo *TRI) {
119 for (auto &MO : FirstI.operands()) {
120 if (!MO.isReg() || !MO.isDef())
122 unsigned R = MO.getReg();
123 if (SecondI.readsRegister(R, TRI))
130 static MachineBasicBlock::iterator moveInstrOut(MachineInstr *MI,
131 MachineBasicBlock::iterator BundleIt, bool Before) {
132 MachineBasicBlock::instr_iterator InsertPt;
134 InsertPt = BundleIt.getInstrIterator();
136 InsertPt = std::next(BundleIt).getInstrIterator();
138 MachineBasicBlock &B = *MI->getParent();
139 // The instruction should at least be bundled with the preceding instruction
140 // (there will always be one, i.e. BUNDLE, if nothing else).
141 assert(MI->isBundledWithPred());
142 if (MI->isBundledWithSucc()) {
143 MI->clearFlag(MachineInstr::BundledSucc);
144 MI->clearFlag(MachineInstr::BundledPred);
146 // If it's not bundled with the successor (i.e. it is the last one
147 // in the bundle), then we can simply unbundle it from the predecessor,
148 // which will take care of updating the predecessor's flag.
149 MI->unbundleFromPred();
151 B.splice(InsertPt, &B, MI);
153 // Get the size of the bundle without asserting.
154 MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator();
155 MachineBasicBlock::const_instr_iterator E = B.instr_end();
157 for (++I; I != E && I->isBundledWithPred(); ++I)
160 // If there are still two or more instructions, then there is nothing
165 // Otherwise, extract the single instruction out and delete the bundle.
166 MachineBasicBlock::iterator NextIt = std::next(BundleIt);
167 MachineInstr *SingleI = BundleIt->getNextNode();
168 SingleI->unbundleFromPred();
169 assert(!SingleI->isBundledWithSucc());
170 BundleIt->eraseFromParent();
175 bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
176 if (DisablePacketizer || skipFunction(*MF.getFunction()))
179 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
180 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
181 auto &MLI = getAnalysis<MachineLoopInfo>();
182 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
183 auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
185 if (EnableGenAllInsnClass)
186 HII->genAllInsnTimingClasses(MF);
188 // Instantiate the packetizer.
189 HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI);
191 // DFA state table should not be empty.
192 assert(Packetizer.getResourceTracker() && "Empty DFA table!");
195 // Loop over all basic blocks and remove KILL pseudo-instructions
196 // These instructions confuse the dependence analysis. Consider:
198 // R0 = KILL R0, D0 (Insn 1)
200 // Here, Insn 1 will result in the dependence graph not emitting an output
201 // dependence between Insn 0 and Insn 2. This can lead to incorrect
204 for (auto &MB : MF) {
206 auto MI = MB.begin();
208 auto NextI = std::next(MI);
217 // Loop over all of the basic blocks.
218 for (auto &MB : MF) {
219 auto Begin = MB.begin(), End = MB.end();
220 while (Begin != End) {
221 // First the first non-boundary starting from the end of the last
222 // scheduling region.
223 MachineBasicBlock::iterator RB = Begin;
224 while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))
226 // First the first boundary starting from the beginning of the new
228 MachineBasicBlock::iterator RE = RB;
229 while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))
231 // Add the scheduling boundary if it's not block end.
234 // If RB == End, then RE == End.
236 Packetizer.PacketizeMIs(&MB, RB, RE);
242 Packetizer.unpacketizeSoloInstrs(MF);
247 // Reserve resources for a constant extender. Trigger an assertion if the
248 // reservation fails.
249 void HexagonPacketizerList::reserveResourcesForConstExt() {
250 if (!tryAllocateResourcesForConstExt(true))
251 llvm_unreachable("Resources not available");
254 bool HexagonPacketizerList::canReserveResourcesForConstExt() {
255 return tryAllocateResourcesForConstExt(false);
258 // Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
259 // return true, otherwise, return false.
260 bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
261 auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc());
262 bool Avail = ResourceTracker->canReserveResources(*ExtMI);
263 if (Reserve && Avail)
264 ResourceTracker->reserveResources(*ExtMI);
265 MF.DeleteMachineInstr(ExtMI);
270 bool HexagonPacketizerList::isCallDependent(const MachineInstr* MI,
271 SDep::Kind DepType, unsigned DepReg) {
272 // Check for LR dependence.
273 if (DepReg == HRI->getRARegister())
276 if (HII->isDeallocRet(MI))
277 if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
280 // Check if this is a predicate dependence.
281 const TargetRegisterClass* RC = HRI->getMinimalPhysRegClass(DepReg);
282 if (RC == &Hexagon::PredRegsRegClass)
285 // Assumes that the first operand of the CALLr is the function address.
286 if (HII->isIndirectCall(MI) && (DepType == SDep::Data)) {
287 MachineOperand MO = MI->getOperand(0);
288 if (MO.isReg() && MO.isUse() && (MO.getReg() == DepReg))
295 static bool isRegDependence(const SDep::Kind DepType) {
296 return DepType == SDep::Data || DepType == SDep::Anti ||
297 DepType == SDep::Output;
300 static bool isDirectJump(const MachineInstr* MI) {
301 return MI->getOpcode() == Hexagon::J2_jump;
304 static bool isSchedBarrier(const MachineInstr* MI) {
305 switch (MI->getOpcode()) {
306 case Hexagon::Y2_barrier:
312 static bool isControlFlow(const MachineInstr* MI) {
313 return (MI->getDesc().isTerminator() || MI->getDesc().isCall());
317 /// Returns true if the instruction modifies a callee-saved register.
318 static bool doesModifyCalleeSavedReg(const MachineInstr *MI,
319 const TargetRegisterInfo *TRI) {
320 const MachineFunction &MF = *MI->getParent()->getParent();
321 for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
322 if (MI->modifiesRegister(*CSR, TRI))
327 // TODO: MI->isIndirectBranch() and IsRegisterJump(MI)
328 // Returns true if an instruction can be promoted to .new predicate or
330 bool HexagonPacketizerList::isNewifiable(const MachineInstr* MI) {
331 return HII->isCondInst(MI) || MI->isReturn() || HII->mayBeNewStore(MI);
334 // Promote an instructiont to its .cur form.
335 // At this time, we have already made a call to canPromoteToDotCur and made
336 // sure that it can *indeed* be promoted.
337 bool HexagonPacketizerList::promoteToDotCur(MachineInstr* MI,
338 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
339 const TargetRegisterClass* RC) {
340 assert(DepType == SDep::Data);
341 int CurOpcode = HII->getDotCurOp(MI);
342 MI->setDesc(HII->get(CurOpcode));
346 void HexagonPacketizerList::cleanUpDotCur() {
347 MachineInstr *MI = NULL;
348 for (auto BI : CurrentPacketMIs) {
349 DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
350 if (BI->getOpcode() == Hexagon::V6_vL32b_cur_ai) {
355 for (auto &MO : BI->operands())
356 if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())
362 // We did not find a use of the CUR, so de-cur it.
363 MI->setDesc(HII->get(Hexagon::V6_vL32b_ai));
364 DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
367 // Check to see if an instruction can be dot cur.
368 bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr *MI,
369 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
370 const TargetRegisterClass *RC) {
371 if (!HII->isV60VectorInstruction(MI))
373 if (!HII->isV60VectorInstruction(&*MII))
376 // Already a dot new instruction.
377 if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
380 if (!HII->mayBeCurLoad(MI))
383 // The "cur value" cannot come from inline asm.
384 if (PacketSU->getInstr()->isInlineAsm())
387 // Make sure candidate instruction uses cur.
388 DEBUG(dbgs() << "Can we DOT Cur Vector MI\n";
390 dbgs() << "in packet\n";);
391 MachineInstr &MJ = *MII;
393 dbgs() << "Checking CUR against ";
396 unsigned DestReg = MI->getOperand(0).getReg();
397 bool FoundMatch = false;
398 for (auto &MO : MJ.operands())
399 if (MO.isReg() && MO.getReg() == DestReg)
404 // Check for existing uses of a vector register within the packet which
405 // would be affected by converting a vector load into .cur formt.
406 for (auto BI : CurrentPacketMIs) {
407 DEBUG(dbgs() << "packet has "; BI->dump(););
408 if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))
412 DEBUG(dbgs() << "Can Dot CUR MI\n"; MI->dump(););
413 // We can convert the opcode into a .cur.
417 // Promote an instruction to its .new form. At this time, we have already
418 // made a call to canPromoteToDotNew and made sure that it can *indeed* be
420 bool HexagonPacketizerList::promoteToDotNew(MachineInstr* MI,
421 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
422 const TargetRegisterClass* RC) {
423 assert (DepType == SDep::Data);
425 if (RC == &Hexagon::PredRegsRegClass)
426 NewOpcode = HII->getDotNewPredOp(MI, MBPI);
428 NewOpcode = HII->getDotNewOp(MI);
429 MI->setDesc(HII->get(NewOpcode));
433 bool HexagonPacketizerList::demoteToDotOld(MachineInstr* MI) {
434 int NewOpcode = HII->getDotOldOp(MI->getOpcode());
435 MI->setDesc(HII->get(NewOpcode));
445 /// Returns true if an instruction is predicated on p0 and false if it's
446 /// predicated on !p0.
447 static PredicateKind getPredicateSense(const MachineInstr &MI,
448 const HexagonInstrInfo *HII) {
449 if (!HII->isPredicated(MI))
451 if (HII->isPredicatedTrue(MI))
456 static const MachineOperand &getPostIncrementOperand(const MachineInstr *MI,
457 const HexagonInstrInfo *HII) {
458 assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
460 // Post Increment means duplicates. Use dense map to find duplicates in the
461 // list. Caution: Densemap initializes with the minimum of 64 buckets,
462 // whereas there are at most 5 operands in the post increment.
463 DenseSet<unsigned> DefRegsSet;
464 for (auto &MO : MI->operands())
465 if (MO.isReg() && MO.isDef())
466 DefRegsSet.insert(MO.getReg());
468 for (auto &MO : MI->operands())
469 if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
473 const MachineOperand &Op1 = MI->getOperand(1);
474 // The 2nd operand is always the post increment operand in load.
475 assert(Op1.isReg() && "Post increment operand has be to a register.");
478 if (MI->getDesc().mayStore()) {
479 const MachineOperand &Op0 = MI->getOperand(0);
480 // The 1st operand is always the post increment operand in store.
481 assert(Op0.isReg() && "Post increment operand has be to a register.");
485 // we should never come here.
486 llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
489 // Get the value being stored.
490 static const MachineOperand& getStoreValueOperand(const MachineInstr *MI) {
491 // value being stored is always the last operand.
492 return MI->getOperand(MI->getNumOperands()-1);
495 static bool isLoadAbsSet(const MachineInstr *MI) {
496 unsigned Opc = MI->getOpcode();
498 case Hexagon::L4_loadrd_ap:
499 case Hexagon::L4_loadrb_ap:
500 case Hexagon::L4_loadrh_ap:
501 case Hexagon::L4_loadrub_ap:
502 case Hexagon::L4_loadruh_ap:
503 case Hexagon::L4_loadri_ap:
509 static const MachineOperand &getAbsSetOperand(const MachineInstr *MI) {
510 assert(isLoadAbsSet(MI));
511 return MI->getOperand(1);
515 // Can be new value store?
516 // Following restrictions are to be respected in convert a store into
517 // a new value store.
518 // 1. If an instruction uses auto-increment, its address register cannot
519 // be a new-value register. Arch Spec 5.4.2.1
520 // 2. If an instruction uses absolute-set addressing mode, its address
521 // register cannot be a new-value register. Arch Spec 5.4.2.1.
522 // 3. If an instruction produces a 64-bit result, its registers cannot be used
523 // as new-value registers. Arch Spec 5.4.2.2.
524 // 4. If the instruction that sets the new-value register is conditional, then
525 // the instruction that uses the new-value register must also be conditional,
526 // and both must always have their predicates evaluate identically.
527 // Arch Spec 5.4.2.3.
528 // 5. There is an implied restriction that a packet cannot have another store,
529 // if there is a new value store in the packet. Corollary: if there is
530 // already a store in a packet, there can not be a new value store.
531 // Arch Spec: 3.4.4.2
532 bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr *MI,
533 const MachineInstr *PacketMI, unsigned DepReg) {
534 // Make sure we are looking at the store, that can be promoted.
535 if (!HII->mayBeNewStore(MI))
538 // Make sure there is dependency and can be new value'd.
539 const MachineOperand &Val = getStoreValueOperand(MI);
540 if (Val.isReg() && Val.getReg() != DepReg)
543 const MCInstrDesc& MCID = PacketMI->getDesc();
545 // First operand is always the result.
546 const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF);
547 // Double regs can not feed into new value store: PRM section: 5.4.2.2.
548 if (PacketRC == &Hexagon::DoubleRegsRegClass)
551 // New-value stores are of class NV (slot 0), dual stores require class ST
552 // in slot 0 (PRM 5.5).
553 for (auto I : CurrentPacketMIs) {
554 SUnit *PacketSU = MIToSUnit.find(I)->second;
555 if (PacketSU->getInstr()->mayStore())
559 // Make sure it's NOT the post increment register that we are going to
561 if (HII->isPostIncrement(MI) &&
562 getPostIncrementOperand(MI, HII).getReg() == DepReg) {
566 if (HII->isPostIncrement(PacketMI) && PacketMI->mayLoad() &&
567 getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) {
568 // If source is post_inc, or absolute-set addressing, it can not feed
569 // into new value store
571 // memw(r30 + #-1404) = r2.new -> can not be new value store
572 // arch spec section: 5.4.2.1.
576 if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg)
579 // If the source that feeds the store is predicated, new value store must
580 // also be predicated.
581 if (HII->isPredicated(*PacketMI)) {
582 if (!HII->isPredicated(*MI))
585 // Check to make sure that they both will have their predicates
586 // evaluate identically.
587 unsigned predRegNumSrc = 0;
588 unsigned predRegNumDst = 0;
589 const TargetRegisterClass* predRegClass = nullptr;
591 // Get predicate register used in the source instruction.
592 for (auto &MO : PacketMI->operands()) {
595 predRegNumSrc = MO.getReg();
596 predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);
597 if (predRegClass == &Hexagon::PredRegsRegClass)
600 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
601 "predicate register not found in a predicated PacketMI instruction");
603 // Get predicate register used in new-value store instruction.
604 for (auto &MO : MI->operands()) {
607 predRegNumDst = MO.getReg();
608 predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);
609 if (predRegClass == &Hexagon::PredRegsRegClass)
612 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
613 "predicate register not found in a predicated MI instruction");
615 // New-value register producer and user (store) need to satisfy these
617 // 1) Both instructions should be predicated on the same register.
618 // 2) If producer of the new-value register is .new predicated then store
619 // should also be .new predicated and if producer is not .new predicated
620 // then store should not be .new predicated.
621 // 3) Both new-value register producer and user should have same predicate
622 // sense, i.e, either both should be negated or both should be non-negated.
623 if (predRegNumDst != predRegNumSrc ||
624 HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||
625 getPredicateSense(*MI, HII) != getPredicateSense(*PacketMI, HII))
629 // Make sure that other than the new-value register no other store instruction
630 // register has been modified in the same packet. Predicate registers can be
631 // modified by they should not be modified between the producer and the store
632 // instruction as it will make them both conditional on different values.
633 // We already know this to be true for all the instructions before and
634 // including PacketMI. Howerver, we need to perform the check for the
635 // remaining instructions in the packet.
637 unsigned StartCheck = 0;
639 for (auto I : CurrentPacketMIs) {
640 SUnit *TempSU = MIToSUnit.find(I)->second;
641 MachineInstr* TempMI = TempSU->getInstr();
643 // Following condition is true for all the instructions until PacketMI is
644 // reached (StartCheck is set to 0 before the for loop).
645 // StartCheck flag is 1 for all the instructions after PacketMI.
646 if (TempMI != PacketMI && !StartCheck) // Start processing only after
647 continue; // encountering PacketMI.
650 if (TempMI == PacketMI) // We don't want to check PacketMI for dependence.
653 for (auto &MO : MI->operands())
654 if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI))
658 // Make sure that for non-POST_INC stores:
659 // 1. The only use of reg is DepReg and no other registers.
660 // This handles V4 base+index registers.
661 // The following store can not be dot new.
662 // Eg. r0 = add(r0, #3)
663 // memw(r1+r0<<#2) = r0
664 if (!HII->isPostIncrement(MI)) {
665 for (unsigned opNum = 0; opNum < MI->getNumOperands()-1; opNum++) {
666 const MachineOperand &MO = MI->getOperand(opNum);
667 if (MO.isReg() && MO.getReg() == DepReg)
672 // If data definition is because of implicit definition of the register,
673 // do not newify the store. Eg.
674 // %R9<def> = ZXTH %R12, %D6<imp-use>, %R12<imp-def>
675 // S2_storerh_io %R8, 2, %R12<kill>; mem:ST2[%scevgep343]
676 for (auto &MO : PacketMI->operands()) {
677 if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
679 unsigned R = MO.getReg();
680 if (R == DepReg || HRI->isSuperRegister(DepReg, R))
684 // Handle imp-use of super reg case. There is a target independent side
685 // change that should prevent this situation but I am handling it for
686 // just-in-case. For example, we cannot newify R2 in the following case:
687 // %R3<def> = A2_tfrsi 0;
688 // S2_storeri_io %R0<kill>, 0, %R2<kill>, %D1<imp-use,kill>;
689 for (auto &MO : MI->operands()) {
690 if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
694 // Can be dot new store.
698 // Can this MI to promoted to either new value store or new value jump.
699 bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr *MI,
700 const SUnit *PacketSU, unsigned DepReg,
701 MachineBasicBlock::iterator &MII) {
702 if (!HII->mayBeNewStore(MI))
705 // Check to see the store can be new value'ed.
706 MachineInstr *PacketMI = PacketSU->getInstr();
707 if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
710 // Check to see the compare/jump can be new value'ed.
711 // This is done as a pass on its own. Don't need to check it here.
715 static bool isImplicitDependency(const MachineInstr *I, unsigned DepReg) {
716 for (auto &MO : I->operands())
717 if (MO.isReg() && MO.isDef() && (MO.getReg() == DepReg) && MO.isImplicit())
722 // Check to see if an instruction can be dot new
723 // There are three kinds.
724 // 1. dot new on predicate - V2/V3/V4
725 // 2. dot new on stores NV/ST - V4
726 // 3. dot new on jump NV/J - V4 -- This is generated in a pass.
727 bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr *MI,
728 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
729 const TargetRegisterClass* RC) {
730 // Already a dot new instruction.
731 if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
734 if (!isNewifiable(MI))
737 const MachineInstr *PI = PacketSU->getInstr();
739 // The "new value" cannot come from inline asm.
740 if (PI->isInlineAsm())
743 // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no
745 if (PI->isImplicitDef())
748 // If dependency is trough an implicitly defined register, we should not
750 if (isImplicitDependency(PI, DepReg))
753 const MCInstrDesc& MCID = PI->getDesc();
754 const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF);
755 if (DisableVecDblNVStores && VecRC == &Hexagon::VecDblRegsRegClass)
759 // bug 5670: until that is fixed
760 // TODO: MI->isIndirectBranch() and IsRegisterJump(MI)
761 if (RC == &Hexagon::PredRegsRegClass)
762 if (HII->isCondInst(MI) || MI->isReturn())
763 return HII->predCanBeUsedAsDotNew(PI, DepReg);
765 if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
768 // Create a dot new machine instruction to see if resources can be
769 // allocated. If not, bail out now.
770 int NewOpcode = HII->getDotNewOp(MI);
771 const MCInstrDesc &D = HII->get(NewOpcode);
772 MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc());
773 bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);
774 MF.DeleteMachineInstr(NewMI);
775 if (!ResourcesAvailable)
778 // New Value Store only. New Value Jump generated as a separate pass.
779 if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
785 // Go through the packet instructions and search for an anti dependency between
786 // them and DepReg from MI. Consider this case:
788 // a) %R1<def> = TFRI_cdNotPt %P3, 2
791 // b) %P0<def> = C2_or %P3<kill>, %P0<kill>
792 // c) %P3<def> = C2_tfrrp %R23
793 // d) %R1<def> = C2_cmovenewit %P3, 4
795 // The P3 from a) and d) will be complements after
796 // a)'s P3 is converted to .new form
797 // Anti-dep between c) and b) is irrelevant for this case
798 bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr* MI,
800 SUnit *PacketSUDep = MIToSUnit.find(MI)->second;
802 for (auto I : CurrentPacketMIs) {
803 // We only care for dependencies to predicated instructions
804 if (!HII->isPredicated(*I))
807 // Scheduling Unit for current insn in the packet
808 SUnit *PacketSU = MIToSUnit.find(I)->second;
810 // Look at dependencies between current members of the packet and
811 // predicate defining instruction MI. Make sure that dependency is
812 // on the exact register we care about.
813 if (PacketSU->isSucc(PacketSUDep)) {
814 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
815 auto &Dep = PacketSU->Succs[i];
816 if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
817 Dep.getReg() == DepReg)
827 /// Gets the predicate register of a predicated instruction.
828 static unsigned getPredicatedRegister(MachineInstr &MI,
829 const HexagonInstrInfo *QII) {
830 /// We use the following rule: The first predicate register that is a use is
831 /// the predicate register of a predicated instruction.
832 assert(QII->isPredicated(MI) && "Must be predicated instruction");
834 for (auto &Op : MI.operands()) {
835 if (Op.isReg() && Op.getReg() && Op.isUse() &&
836 Hexagon::PredRegsRegClass.contains(Op.getReg()))
840 llvm_unreachable("Unknown instruction operand layout");
844 // Given two predicated instructions, this function detects whether
845 // the predicates are complements.
846 bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
848 // If we don't know the predicate sense of the instructions bail out early, we
850 if (getPredicateSense(MI1, HII) == PK_Unknown ||
851 getPredicateSense(MI2, HII) == PK_Unknown)
854 // Scheduling unit for candidate.
855 SUnit *SU = MIToSUnit[&MI1];
857 // One corner case deals with the following scenario:
859 // a) %R24<def> = A2_tfrt %P0, %R25
862 // b) %R25<def> = A2_tfrf %P0, %R24
863 // c) %P0<def> = C2_cmpeqi %R26, 1
866 // On general check a) and b) are complements, but presence of c) will
867 // convert a) to .new form, and then it is not a complement.
868 // We attempt to detect it by analyzing existing dependencies in the packet.
870 // Analyze relationships between all existing members of the packet.
871 // Look for Anti dependecy on the same predicate reg as used in the
873 for (auto I : CurrentPacketMIs) {
874 // Scheduling Unit for current insn in the packet.
875 SUnit *PacketSU = MIToSUnit.find(I)->second;
877 // If this instruction in the packet is succeeded by the candidate...
878 if (PacketSU->isSucc(SU)) {
879 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
880 auto Dep = PacketSU->Succs[i];
881 // The corner case exist when there is true data dependency between
882 // candidate and one of current packet members, this dep is on
883 // predicate reg, and there already exist anti dep on the same pred in
885 if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
886 Hexagon::PredRegsRegClass.contains(Dep.getReg())) {
887 // Here I know that I is predicate setting instruction with true
888 // data dep to candidate on the register we care about - c) in the
889 // above example. Now I need to see if there is an anti dependency
890 // from c) to any other instruction in the same packet on the pred
892 if (restrictingDepExistInPacket(I, Dep.getReg()))
899 // If the above case does not apply, check regular complement condition.
900 // Check that the predicate register is the same and that the predicate
901 // sense is different We also need to differentiate .old vs. .new: !p0
902 // is not complementary to p0.new.
903 unsigned PReg1 = getPredicatedRegister(MI1, HII);
904 unsigned PReg2 = getPredicatedRegister(MI2, HII);
905 return PReg1 == PReg2 &&
906 Hexagon::PredRegsRegClass.contains(PReg1) &&
907 Hexagon::PredRegsRegClass.contains(PReg2) &&
908 getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) &&
909 HII->isDotNewInst(&MI1) == HII->isDotNewInst(&MI2);
912 // Initialize packetizer flags.
913 void HexagonPacketizerList::initPacketizerState() {
915 PromotedToDotNew = false;
916 GlueToNewValueJump = false;
917 GlueAllocframeStore = false;
918 FoundSequentialDependence = false;
921 // Ignore bundling of pseudo instructions.
922 bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI,
923 const MachineBasicBlock *) {
924 if (MI.isDebugValue())
927 if (MI.isCFIInstruction())
930 // We must print out inline assembly.
931 if (MI.isInlineAsm())
934 if (MI.isImplicitDef())
937 // We check if MI has any functional units mapped to it. If it doesn't,
938 // we ignore the instruction.
939 const MCInstrDesc& TID = MI.getDesc();
940 auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass());
941 unsigned FuncUnits = IS->getUnits();
945 bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
946 if (MI.isEHLabel() || MI.isCFIInstruction())
949 // Consider inline asm to not be a solo instruction by default.
950 // Inline asm will be put in a packet temporarily, but then it will be
951 // removed, and placed outside of the packet (before or after, depending
952 // on dependencies). This is to reduce the impact of inline asm as a
953 // "packet splitting" instruction.
954 if (MI.isInlineAsm() && !ScheduleInlineAsm)
957 // From Hexagon V4 Programmer's Reference Manual 3.4.4 Grouping constraints:
958 // trap, pause, barrier, icinva, isync, and syncht are solo instructions.
959 // They must not be grouped with other instructions in a packet.
960 if (isSchedBarrier(&MI))
963 if (HII->isSolo(&MI))
966 if (MI.getOpcode() == Hexagon::A2_nop)
973 // Quick check if instructions MI and MJ cannot coexist in the same packet.
974 // Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
975 // but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
976 // For full test call this function twice:
977 // cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
978 // Doing the test only one way saves the amount of code in this function,
979 // since every test would need to be repeated with the MI and MJ reversed.
980 static bool cannotCoexistAsymm(const MachineInstr *MI, const MachineInstr *MJ,
981 const HexagonInstrInfo &HII) {
982 const MachineFunction *MF = MI->getParent()->getParent();
983 if (MF->getSubtarget<HexagonSubtarget>().hasV60TOpsOnly() &&
984 HII.isHVXMemWithAIndirect(MI, MJ))
987 // An inline asm cannot be together with a branch, because we may not be
988 // able to remove the asm out after packetizing (i.e. if the asm must be
989 // moved past the bundle). Similarly, two asms cannot be together to avoid
990 // complications when determining their relative order outside of a bundle.
991 if (MI->isInlineAsm())
992 return MJ->isInlineAsm() || MJ->isBranch() || MJ->isBarrier() ||
993 MJ->isCall() || MJ->isTerminator();
995 // "False" really means that the quick check failed to determine if
996 // I and J cannot coexist.
1001 // Full, symmetric check.
1002 bool HexagonPacketizerList::cannotCoexist(const MachineInstr *MI,
1003 const MachineInstr *MJ) {
1004 return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII);
1007 void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) {
1008 for (auto &B : MF) {
1009 MachineBasicBlock::iterator BundleIt;
1010 MachineBasicBlock::instr_iterator NextI;
1011 for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) {
1012 NextI = std::next(I);
1013 MachineInstr *MI = &*I;
1016 if (!MI->isInsideBundle())
1019 // Decide on where to insert the instruction that we are pulling out.
1020 // Debug instructions always go before the bundle, but the placement of
1021 // INLINE_ASM depends on potential dependencies. By default, try to
1022 // put it before the bundle, but if the asm writes to a register that
1023 // other instructions in the bundle read, then we need to place it
1024 // after the bundle (to preserve the bundle semantics).
1025 bool InsertBeforeBundle;
1026 if (MI->isInlineAsm())
1027 InsertBeforeBundle = !hasWriteToReadDep(*MI, *BundleIt, HRI);
1028 else if (MI->isDebugValue())
1029 InsertBeforeBundle = true;
1033 BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle);
1038 // Check if a given instruction is of class "system".
1039 static bool isSystemInstr(const MachineInstr *MI) {
1040 unsigned Opc = MI->getOpcode();
1042 case Hexagon::Y2_barrier:
1043 case Hexagon::Y2_dcfetchbo:
1049 bool HexagonPacketizerList::hasDeadDependence(const MachineInstr *I,
1050 const MachineInstr *J) {
1051 // The dependence graph may not include edges between dead definitions,
1052 // so without extra checks, we could end up packetizing two instruction
1053 // defining the same (dead) register.
1054 if (I->isCall() || J->isCall())
1056 if (HII->isPredicated(*I) || HII->isPredicated(*J))
1059 BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
1060 for (auto &MO : I->operands()) {
1061 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1063 DeadDefs[MO.getReg()] = true;
1066 for (auto &MO : J->operands()) {
1067 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1069 unsigned R = MO.getReg();
1070 if (R != Hexagon::USR_OVF && DeadDefs[R])
1076 bool HexagonPacketizerList::hasControlDependence(const MachineInstr *I,
1077 const MachineInstr *J) {
1078 // A save callee-save register function call can only be in a packet
1079 // with instructions that don't write to the callee-save registers.
1080 if ((HII->isSaveCalleeSavedRegsCall(I) &&
1081 doesModifyCalleeSavedReg(J, HRI)) ||
1082 (HII->isSaveCalleeSavedRegsCall(J) &&
1083 doesModifyCalleeSavedReg(I, HRI)))
1086 // Two control flow instructions cannot go in the same packet.
1087 if (isControlFlow(I) && isControlFlow(J))
1090 // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
1091 // contain a speculative indirect jump,
1092 // a new-value compare jump or a dealloc_return.
1093 auto isBadForLoopN = [this] (const MachineInstr *MI) -> bool {
1094 if (MI->isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
1096 if (HII->isPredicated(*MI) && HII->isPredicatedNew(*MI) && HII->isJumpR(MI))
1101 if (HII->isLoopN(I) && isBadForLoopN(J))
1103 if (HII->isLoopN(J) && isBadForLoopN(I))
1106 // dealloc_return cannot appear in the same packet as a conditional or
1107 // unconditional jump.
1108 return HII->isDeallocRet(I) &&
1109 (J->isBranch() || J->isCall() || J->isBarrier());
1112 bool HexagonPacketizerList::hasV4SpecificDependence(const MachineInstr *I,
1113 const MachineInstr *J) {
1114 bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J);
1115 bool StoreI = I->mayStore(), StoreJ = J->mayStore();
1116 if ((SysI && StoreJ) || (SysJ && StoreI))
1119 if (StoreI && StoreJ) {
1120 if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))
1123 // A memop cannot be in the same packet with another memop or a store.
1124 // Two stores can be together, but here I and J cannot both be stores.
1125 bool MopStI = HII->isMemOp(I) || StoreI;
1126 bool MopStJ = HII->isMemOp(J) || StoreJ;
1127 if (MopStI && MopStJ)
1131 return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));
1134 // SUI is the current instruction that is out side of the current packet.
1135 // SUJ is the current instruction inside the current packet against which that
1136 // SUI will be packetized.
1137 bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
1138 MachineInstr *I = SUI->getInstr();
1139 MachineInstr *J = SUJ->getInstr();
1140 assert(I && J && "Unable to packetize null instruction!");
1142 // Clear IgnoreDepMIs when Packet starts.
1143 if (CurrentPacketMIs.size() == 1)
1144 IgnoreDepMIs.clear();
1146 MachineBasicBlock::iterator II = I;
1147 const unsigned FrameSize = MF.getFrameInfo()->getStackSize();
1149 // Solo instructions cannot go in the packet.
1150 assert(!isSoloInstruction(*I) && "Unexpected solo instr!");
1152 if (cannotCoexist(I, J))
1155 Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
1159 // V4 allows dual stores. It does not allow second store, if the first
1160 // store is not in SLOT0. New value store, new value jump, dealloc_return
1161 // and memop always take SLOT0. Arch spec 3.4.4.2.
1162 Dependence = hasV4SpecificDependence(I, J);
1166 // If an instruction feeds new value jump, glue it.
1167 MachineBasicBlock::iterator NextMII = I;
1169 if (NextMII != I->getParent()->end() && HII->isNewValueJump(&*NextMII)) {
1170 MachineInstr &NextMI = *NextMII;
1172 bool secondRegMatch = false;
1173 const MachineOperand &NOp0 = NextMI.getOperand(0);
1174 const MachineOperand &NOp1 = NextMI.getOperand(1);
1176 if (NOp1.isReg() && I->getOperand(0).getReg() == NOp1.getReg())
1177 secondRegMatch = true;
1179 for (auto I : CurrentPacketMIs) {
1180 SUnit *PacketSU = MIToSUnit.find(I)->second;
1181 MachineInstr *PI = PacketSU->getInstr();
1182 // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
1188 // 1. Packet does not have a store in it.
1189 // 2. If the first operand of the nvj is newified, and the second
1190 // operand is also a reg, it (second reg) is not defined in
1192 // 3. If the second operand of the nvj is newified, (which means
1193 // first operand is also a reg), first reg is not defined in
1195 if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||
1201 const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
1202 if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) {
1210 GlueToNewValueJump = true;
1213 // There no dependency between a prolog instruction and its successor.
1214 if (!SUJ->isSucc(SUI))
1217 for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
1218 if (FoundSequentialDependence)
1221 if (SUJ->Succs[i].getSUnit() != SUI)
1224 SDep::Kind DepType = SUJ->Succs[i].getKind();
1225 // For direct calls:
1226 // Ignore register dependences for call instructions for packetization
1227 // purposes except for those due to r31 and predicate registers.
1229 // For indirect calls:
1230 // Same as direct calls + check for true dependences to the register
1231 // used in the indirect call.
1233 // We completely ignore Order dependences for call instructions.
1236 // Ignore register dependences for return instructions like jumpr,
1237 // dealloc return unless we have dependencies on the explicit uses
1238 // of the registers used by jumpr (like r31) or dealloc return
1239 // (like r29 or r30).
1241 // TODO: Currently, jumpr is handling only return of r31. So, the
1242 // following logic (specificaly isCallDependent) is working fine.
1243 // We need to enable jumpr for register other than r31 and then,
1244 // we need to rework the last part, where it handles indirect call
1245 // of that (isCallDependent) function. Bug 6216 is opened for this.
1246 unsigned DepReg = 0;
1247 const TargetRegisterClass *RC = nullptr;
1248 if (DepType == SDep::Data) {
1249 DepReg = SUJ->Succs[i].getReg();
1250 RC = HRI->getMinimalPhysRegClass(DepReg);
1253 if (I->isCall() || I->isReturn() || HII->isTailCall(I)) {
1254 if (!isRegDependence(DepType))
1256 if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg()))
1260 if (DepType == SDep::Data) {
1261 if (canPromoteToDotCur(J, SUJ, DepReg, II, RC))
1262 if (promoteToDotCur(J, DepType, II, RC))
1266 // Data dpendence ok if we have load.cur.
1267 if (DepType == SDep::Data && HII->isDotCurInst(J)) {
1268 if (HII->isV60VectorInstruction(I))
1272 // For instructions that can be promoted to dot-new, try to promote.
1273 if (DepType == SDep::Data) {
1274 if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) {
1275 if (promoteToDotNew(I, DepType, II, RC)) {
1276 PromotedToDotNew = true;
1280 if (HII->isNewValueJump(I))
1284 // For predicated instructions, if the predicates are complements then
1285 // there can be no dependence.
1286 if (HII->isPredicated(*I) && HII->isPredicated(*J) &&
1287 arePredicatesComplements(*I, *J)) {
1288 // Not always safe to do this translation.
1289 // DAG Builder attempts to reduce dependence edges using transitive
1290 // nature of dependencies. Here is an example:
1292 // r0 = tfr_pt ... (1)
1293 // r0 = tfr_pf ... (2)
1294 // r0 = tfr_pt ... (3)
1296 // There will be an output dependence between (1)->(2) and (2)->(3).
1297 // However, there is no dependence edge between (1)->(3). This results
1298 // in all 3 instructions going in the same packet. We ignore dependce
1299 // only once to avoid this situation.
1300 auto Itr = std::find(IgnoreDepMIs.begin(), IgnoreDepMIs.end(), J);
1301 if (Itr != IgnoreDepMIs.end()) {
1305 IgnoreDepMIs.push_back(I);
1309 // Ignore Order dependences between unconditional direct branches
1310 // and non-control-flow instructions.
1311 if (isDirectJump(I) && !J->isBranch() && !J->isCall() &&
1312 DepType == SDep::Order)
1315 // Ignore all dependences for jumps except for true and output
1317 if (I->isConditionalBranch() && DepType != SDep::Data &&
1318 DepType != SDep::Output)
1321 // Ignore output dependences due to superregs. We can write to two
1322 // different subregisters of R1:0 for instance in the same cycle.
1324 // If neither I nor J defines DepReg, then this is a superfluous output
1325 // dependence. The dependence must be of the form:
1328 // and there is an output dependence between the two instructions with
1330 // We want to ignore these dependences. Ideally, the dependence
1331 // constructor should annotate such dependences. We can then avoid this
1332 // relatively expensive check.
1334 if (DepType == SDep::Output) {
1335 // DepReg is the register that's responsible for the dependence.
1336 unsigned DepReg = SUJ->Succs[i].getReg();
1338 // Check if I and J really defines DepReg.
1339 if (!I->definesRegister(DepReg) && !J->definesRegister(DepReg))
1341 FoundSequentialDependence = true;
1345 // For Order dependences:
1346 // 1. On V4 or later, volatile loads/stores can be packetized together,
1347 // unless other rules prevent is.
1348 // 2. Store followed by a load is not allowed.
1349 // 3. Store followed by a store is only valid on V4 or later.
1350 // 4. Load followed by any memory operation is allowed.
1351 if (DepType == SDep::Order) {
1352 if (!PacketizeVolatiles) {
1353 bool OrdRefs = I->hasOrderedMemoryRef() || J->hasOrderedMemoryRef();
1355 FoundSequentialDependence = true;
1359 // J is first, I is second.
1360 bool LoadJ = J->mayLoad(), StoreJ = J->mayStore();
1361 bool LoadI = I->mayLoad(), StoreI = I->mayStore();
1363 // Two stores are only allowed on V4+. Load following store is never
1366 FoundSequentialDependence = true;
1369 } else if (!LoadJ || (!LoadI && !StoreI)) {
1370 // If J is neither load nor store, assume a dependency.
1371 // If J is a load, but I is neither, also assume a dependency.
1372 FoundSequentialDependence = true;
1375 // Store followed by store: not OK on V2.
1376 // Store followed by load: not OK on all.
1377 // Load followed by store: OK on all.
1378 // Load followed by load: OK on all.
1382 // For V4, special case ALLOCFRAME. Even though there is dependency
1383 // between ALLOCFRAME and subsequent store, allow it to be packetized
1384 // in a same packet. This implies that the store is using the caller's
1385 // SP. Hence, offset needs to be updated accordingly.
1386 if (DepType == SDep::Data && J->getOpcode() == Hexagon::S2_allocframe) {
1387 unsigned Opc = I->getOpcode();
1389 case Hexagon::S2_storerd_io:
1390 case Hexagon::S2_storeri_io:
1391 case Hexagon::S2_storerh_io:
1392 case Hexagon::S2_storerb_io:
1393 if (I->getOperand(0).getReg() == HRI->getStackRegister()) {
1394 int64_t Imm = I->getOperand(1).getImm();
1395 int64_t NewOff = Imm - (FrameSize + HEXAGON_LRFP_SIZE);
1396 if (HII->isValidOffset(Opc, NewOff)) {
1397 GlueAllocframeStore = true;
1398 // Since this store is to be glued with allocframe in the same
1399 // packet, it will use SP of the previous stack frame, i.e.
1400 // caller's SP. Therefore, we need to recalculate offset
1401 // according to this change.
1402 I->getOperand(1).setImm(NewOff);
1411 // There are certain anti-dependencies that cannot be ignored.
1413 // J2_call ... %R0<imp-def> ; SUJ
1415 // Those cannot be packetized together, since the call will observe
1416 // the effect of the assignment to R0.
1417 if (DepType == SDep::Anti && J->isCall()) {
1418 // Check if I defines any volatile register. We should also check
1419 // registers that the call may read, but these happen to be a
1420 // subset of the volatile register set.
1421 for (const MCPhysReg *P = J->getDesc().ImplicitDefs; P && *P; ++P) {
1422 if (!I->modifiesRegister(*P, HRI))
1424 FoundSequentialDependence = true;
1429 // Skip over remaining anti-dependences. Two instructions that are
1430 // anti-dependent can share a packet, since in most such cases all
1431 // operands are read before any modifications take place.
1432 // The exceptions are branch and call instructions, since they are
1433 // executed after all other instructions have completed (at least
1435 if (DepType != SDep::Anti) {
1436 FoundSequentialDependence = true;
1441 if (FoundSequentialDependence) {
1449 bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
1450 MachineInstr *I = SUI->getInstr();
1451 MachineInstr *J = SUJ->getInstr();
1452 assert(I && J && "Unable to packetize null instruction!");
1454 if (cannotCoexist(I, J))
1460 // Check if the instruction was promoted to a dot-new. If so, demote it
1461 // back into a dot-old.
1462 if (PromotedToDotNew)
1466 // Check if the instruction (must be a store) was glued with an allocframe
1467 // instruction. If so, restore its offset to its original value, i.e. use
1468 // current SP instead of caller's SP.
1469 if (GlueAllocframeStore) {
1470 unsigned FrameSize = MF.getFrameInfo()->getStackSize();
1471 MachineOperand &MOff = I->getOperand(1);
1472 MOff.setImm(MOff.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
1477 MachineBasicBlock::iterator
1478 HexagonPacketizerList::addToPacket(MachineInstr &MI) {
1479 MachineBasicBlock::iterator MII = MI;
1480 MachineBasicBlock *MBB = MI.getParent();
1481 if (MI.isImplicitDef()) {
1482 unsigned R = MI.getOperand(0).getReg();
1483 if (Hexagon::IntRegsRegClass.contains(R)) {
1484 MCSuperRegIterator S(R, HRI, false);
1485 MI.addOperand(MachineOperand::CreateReg(*S, true, true));
1489 assert(ResourceTracker->canReserveResources(MI));
1491 bool ExtMI = HII->isExtended(&MI) || HII->isConstExtended(&MI);
1494 if (GlueToNewValueJump) {
1495 MachineInstr &NvjMI = *++MII;
1496 // We need to put both instructions in the same packet: MI and NvjMI.
1497 // Either of them can require a constant extender. Try to add both to
1498 // the current packet, and if that fails, end the packet and start a
1500 ResourceTracker->reserveResources(MI);
1502 Good = tryAllocateResourcesForConstExt(true);
1504 bool ExtNvjMI = HII->isExtended(&NvjMI) || HII->isConstExtended(&NvjMI);
1506 if (ResourceTracker->canReserveResources(NvjMI))
1507 ResourceTracker->reserveResources(NvjMI);
1511 if (Good && ExtNvjMI)
1512 Good = tryAllocateResourcesForConstExt(true);
1516 assert(ResourceTracker->canReserveResources(MI));
1517 ResourceTracker->reserveResources(MI);
1519 assert(canReserveResourcesForConstExt());
1520 tryAllocateResourcesForConstExt(true);
1522 assert(ResourceTracker->canReserveResources(NvjMI));
1523 ResourceTracker->reserveResources(NvjMI);
1525 assert(canReserveResourcesForConstExt());
1526 reserveResourcesForConstExt();
1529 CurrentPacketMIs.push_back(&MI);
1530 CurrentPacketMIs.push_back(&NvjMI);
1534 ResourceTracker->reserveResources(MI);
1535 if (ExtMI && !tryAllocateResourcesForConstExt(true)) {
1537 if (PromotedToDotNew)
1538 demoteToDotOld(&MI);
1539 ResourceTracker->reserveResources(MI);
1540 reserveResourcesForConstExt();
1543 CurrentPacketMIs.push_back(&MI);
1547 void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB,
1548 MachineBasicBlock::iterator MI) {
1549 OldPacketMIs = CurrentPacketMIs;
1550 VLIWPacketizerList::endPacket(MBB, MI);
1553 bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
1554 return !producesStall(&MI);
1558 // Return true when ConsMI uses a register defined by ProdMI.
1559 static bool isDependent(const MachineInstr *ProdMI,
1560 const MachineInstr *ConsMI) {
1561 if (!ProdMI->getOperand(0).isReg())
1563 unsigned DstReg = ProdMI->getOperand(0).getReg();
1565 for (auto &Op : ConsMI->operands())
1566 if (Op.isReg() && Op.isUse() && Op.getReg() == DstReg)
1567 // The MIs depend on each other.
1573 // V60 forward scheduling.
1574 bool HexagonPacketizerList::producesStall(const MachineInstr *I) {
1575 // Check whether the previous packet is in a different loop. If this is the
1576 // case, there is little point in trying to avoid a stall because that would
1577 // favor the rare case (loop entry) over the common case (loop iteration).
1579 // TODO: We should really be able to check all the incoming edges if this is
1580 // the first packet in a basic block, so we can avoid stalls from the loop
1582 if (!OldPacketMIs.empty()) {
1583 auto *OldBB = OldPacketMIs.front()->getParent();
1584 auto *ThisBB = I->getParent();
1585 if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB))
1589 // Check for stall between two vector instructions.
1590 if (HII->isV60VectorInstruction(I)) {
1591 for (auto J : OldPacketMIs) {
1592 if (!HII->isV60VectorInstruction(J))
1594 if (isDependent(J, I) && !HII->isVecUsableNextPacket(J, I))
1600 // Check for stall between two scalar instructions. First, check that
1601 // there is no definition of a use in the current packet, because it
1602 // may be a candidate for .new.
1603 for (auto J : CurrentPacketMIs)
1604 if (!HII->isV60VectorInstruction(J) && isDependent(J, I))
1607 // Check for stall between I and instructions in the previous packet.
1608 if (MF.getSubtarget<HexagonSubtarget>().useBSBScheduling()) {
1609 for (auto J : OldPacketMIs) {
1610 if (HII->isV60VectorInstruction(J))
1612 if (!HII->isLateInstrFeedsEarlyInstr(J, I))
1614 if (isDependent(J, I) && !HII->canExecuteInBundle(J, I))
1623 //===----------------------------------------------------------------------===//
1624 // Public Constructor Functions
1625 //===----------------------------------------------------------------------===//
1627 FunctionPass *llvm::createHexagonPacketizer() {
1628 return new HexagonPacketizer();