1 //===----- HexagonPacketizer.cpp - vliw packetizer ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This implements a simple VLIW packetizer using DFA. The packetizer works on
11 // machine basic blocks. For each instruction I in BB, the packetizer consults
12 // the DFA to see if machine resources are available to execute I. If so, the
13 // packetizer checks if I depends on any instruction J in the current packet.
14 // If no dependency is found, I is added to current packet and machine resource
15 // is marked as taken. If any dependency is found, a target API call is made to
16 // prune the dependence.
18 //===----------------------------------------------------------------------===//
19 #include "HexagonRegisterInfo.h"
20 #include "HexagonSubtarget.h"
21 #include "HexagonTargetMachine.h"
22 #include "HexagonVLIWPacketizer.h"
23 #include "llvm/Analysis/AliasAnalysis.h"
24 #include "llvm/CodeGen/MachineDominators.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineLoopInfo.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/Passes.h"
29 #include "llvm/Support/CommandLine.h"
30 #include "llvm/Support/Debug.h"
34 #define DEBUG_TYPE "packets"
36 static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden,
37 cl::ZeroOrMore, cl::init(false),
38 cl::desc("Disable Hexagon packetizer pass"));
40 static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles",
41 cl::ZeroOrMore, cl::Hidden, cl::init(true),
42 cl::desc("Allow non-solo packetization of volatile memory references"));
44 static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false),
45 cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC"));
47 static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores",
48 cl::init(false), cl::Hidden, cl::ZeroOrMore,
49 cl::desc("Disable vector double new-value-stores"));
51 extern cl::opt<bool> ScheduleInlineAsm;
54 FunctionPass *createHexagonPacketizer();
55 void initializeHexagonPacketizerPass(PassRegistry&);
60 class HexagonPacketizer : public MachineFunctionPass {
63 HexagonPacketizer() : MachineFunctionPass(ID) {
64 initializeHexagonPacketizerPass(*PassRegistry::getPassRegistry());
67 void getAnalysisUsage(AnalysisUsage &AU) const override {
69 AU.addRequired<AAResultsWrapperPass>();
70 AU.addRequired<MachineBranchProbabilityInfo>();
71 AU.addRequired<MachineDominatorTree>();
72 AU.addRequired<MachineLoopInfo>();
73 AU.addPreserved<MachineDominatorTree>();
74 AU.addPreserved<MachineLoopInfo>();
75 MachineFunctionPass::getAnalysisUsage(AU);
77 StringRef getPassName() const override { return "Hexagon Packetizer"; }
78 bool runOnMachineFunction(MachineFunction &Fn) override;
79 MachineFunctionProperties getRequiredProperties() const override {
80 return MachineFunctionProperties().set(
81 MachineFunctionProperties::Property::NoVRegs);
85 const HexagonInstrInfo *HII;
86 const HexagonRegisterInfo *HRI;
89 char HexagonPacketizer::ID = 0;
92 INITIALIZE_PASS_BEGIN(HexagonPacketizer, "packets", "Hexagon Packetizer",
94 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
95 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
96 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
97 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
98 INITIALIZE_PASS_END(HexagonPacketizer, "packets", "Hexagon Packetizer",
101 HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF,
102 MachineLoopInfo &MLI, AliasAnalysis *AA,
103 const MachineBranchProbabilityInfo *MBPI)
104 : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI) {
105 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
106 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
108 addMutation(make_unique<HexagonSubtarget::HexagonDAGMutation>());
111 // Check if FirstI modifies a register that SecondI reads.
112 static bool hasWriteToReadDep(const MachineInstr &FirstI,
113 const MachineInstr &SecondI,
114 const TargetRegisterInfo *TRI) {
115 for (auto &MO : FirstI.operands()) {
116 if (!MO.isReg() || !MO.isDef())
118 unsigned R = MO.getReg();
119 if (SecondI.readsRegister(R, TRI))
126 static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI,
127 MachineBasicBlock::iterator BundleIt, bool Before) {
128 MachineBasicBlock::instr_iterator InsertPt;
130 InsertPt = BundleIt.getInstrIterator();
132 InsertPt = std::next(BundleIt).getInstrIterator();
134 MachineBasicBlock &B = *MI.getParent();
135 // The instruction should at least be bundled with the preceding instruction
136 // (there will always be one, i.e. BUNDLE, if nothing else).
137 assert(MI.isBundledWithPred());
138 if (MI.isBundledWithSucc()) {
139 MI.clearFlag(MachineInstr::BundledSucc);
140 MI.clearFlag(MachineInstr::BundledPred);
142 // If it's not bundled with the successor (i.e. it is the last one
143 // in the bundle), then we can simply unbundle it from the predecessor,
144 // which will take care of updating the predecessor's flag.
145 MI.unbundleFromPred();
147 B.splice(InsertPt, &B, MI.getIterator());
149 // Get the size of the bundle without asserting.
150 MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator();
151 MachineBasicBlock::const_instr_iterator E = B.instr_end();
153 for (++I; I != E && I->isBundledWithPred(); ++I)
156 // If there are still two or more instructions, then there is nothing
161 // Otherwise, extract the single instruction out and delete the bundle.
162 MachineBasicBlock::iterator NextIt = std::next(BundleIt);
163 MachineInstr &SingleI = *BundleIt->getNextNode();
164 SingleI.unbundleFromPred();
165 assert(!SingleI.isBundledWithSucc());
166 BundleIt->eraseFromParent();
171 bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
172 if (DisablePacketizer || skipFunction(*MF.getFunction()))
175 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
176 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
177 auto &MLI = getAnalysis<MachineLoopInfo>();
178 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
179 auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
181 if (EnableGenAllInsnClass)
182 HII->genAllInsnTimingClasses(MF);
184 // Instantiate the packetizer.
185 HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI);
187 // DFA state table should not be empty.
188 assert(Packetizer.getResourceTracker() && "Empty DFA table!");
191 // Loop over all basic blocks and remove KILL pseudo-instructions
192 // These instructions confuse the dependence analysis. Consider:
194 // R0 = KILL R0, D0 (Insn 1)
196 // Here, Insn 1 will result in the dependence graph not emitting an output
197 // dependence between Insn 0 and Insn 2. This can lead to incorrect
200 for (auto &MB : MF) {
202 auto MI = MB.begin();
204 auto NextI = std::next(MI);
213 // Loop over all of the basic blocks.
214 for (auto &MB : MF) {
215 auto Begin = MB.begin(), End = MB.end();
216 while (Begin != End) {
217 // First the first non-boundary starting from the end of the last
218 // scheduling region.
219 MachineBasicBlock::iterator RB = Begin;
220 while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))
222 // First the first boundary starting from the beginning of the new
224 MachineBasicBlock::iterator RE = RB;
225 while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))
227 // Add the scheduling boundary if it's not block end.
230 // If RB == End, then RE == End.
232 Packetizer.PacketizeMIs(&MB, RB, RE);
238 Packetizer.unpacketizeSoloInstrs(MF);
243 // Reserve resources for a constant extender. Trigger an assertion if the
244 // reservation fails.
245 void HexagonPacketizerList::reserveResourcesForConstExt() {
246 if (!tryAllocateResourcesForConstExt(true))
247 llvm_unreachable("Resources not available");
250 bool HexagonPacketizerList::canReserveResourcesForConstExt() {
251 return tryAllocateResourcesForConstExt(false);
254 // Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
255 // return true, otherwise, return false.
256 bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
257 auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc());
258 bool Avail = ResourceTracker->canReserveResources(*ExtMI);
259 if (Reserve && Avail)
260 ResourceTracker->reserveResources(*ExtMI);
261 MF.DeleteMachineInstr(ExtMI);
266 bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI,
267 SDep::Kind DepType, unsigned DepReg) {
268 // Check for LR dependence.
269 if (DepReg == HRI->getRARegister())
272 if (HII->isDeallocRet(MI))
273 if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
276 // Check if this is a predicate dependence.
277 const TargetRegisterClass* RC = HRI->getMinimalPhysRegClass(DepReg);
278 if (RC == &Hexagon::PredRegsRegClass)
281 // Assumes that the first operand of the CALLr is the function address.
282 if (HII->isIndirectCall(MI) && (DepType == SDep::Data)) {
283 const MachineOperand MO = MI.getOperand(0);
284 if (MO.isReg() && MO.isUse() && (MO.getReg() == DepReg))
288 if (HII->isJumpR(MI)) {
289 const MachineOperand &MO = HII->isPredicated(MI) ? MI.getOperand(1)
291 assert(MO.isReg() && MO.isUse());
292 if (MO.getReg() == DepReg)
298 static bool isRegDependence(const SDep::Kind DepType) {
299 return DepType == SDep::Data || DepType == SDep::Anti ||
300 DepType == SDep::Output;
303 static bool isDirectJump(const MachineInstr &MI) {
304 return MI.getOpcode() == Hexagon::J2_jump;
307 static bool isSchedBarrier(const MachineInstr &MI) {
308 switch (MI.getOpcode()) {
309 case Hexagon::Y2_barrier:
315 static bool isControlFlow(const MachineInstr &MI) {
316 return MI.getDesc().isTerminator() || MI.getDesc().isCall();
320 /// Returns true if the instruction modifies a callee-saved register.
321 static bool doesModifyCalleeSavedReg(const MachineInstr &MI,
322 const TargetRegisterInfo *TRI) {
323 const MachineFunction &MF = *MI.getParent()->getParent();
324 for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
325 if (MI.modifiesRegister(*CSR, TRI))
330 // Returns true if an instruction can be promoted to .new predicate or
332 bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI,
333 const TargetRegisterClass *NewRC) {
334 // Vector stores can be predicated, and can be new-value stores, but
335 // they cannot be predicated on a .new predicate value.
336 if (NewRC == &Hexagon::PredRegsRegClass)
337 if (HII->isV60VectorInstruction(MI) && MI.mayStore())
339 return HII->isCondInst(MI) || HII->isJumpR(MI) || MI.isReturn() ||
340 HII->mayBeNewStore(MI);
343 // Promote an instructiont to its .cur form.
344 // At this time, we have already made a call to canPromoteToDotCur and made
345 // sure that it can *indeed* be promoted.
346 bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI,
347 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
348 const TargetRegisterClass* RC) {
349 assert(DepType == SDep::Data);
350 int CurOpcode = HII->getDotCurOp(MI);
351 MI.setDesc(HII->get(CurOpcode));
355 void HexagonPacketizerList::cleanUpDotCur() {
356 MachineInstr *MI = nullptr;
357 for (auto BI : CurrentPacketMIs) {
358 DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
359 if (BI->getOpcode() == Hexagon::V6_vL32b_cur_ai) {
364 for (auto &MO : BI->operands())
365 if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())
371 // We did not find a use of the CUR, so de-cur it.
372 MI->setDesc(HII->get(Hexagon::V6_vL32b_ai));
373 DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
376 // Check to see if an instruction can be dot cur.
377 bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI,
378 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
379 const TargetRegisterClass *RC) {
380 if (!HII->isV60VectorInstruction(MI))
382 if (!HII->isV60VectorInstruction(*MII))
385 // Already a dot new instruction.
386 if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
389 if (!HII->mayBeCurLoad(MI))
392 // The "cur value" cannot come from inline asm.
393 if (PacketSU->getInstr()->isInlineAsm())
396 // Make sure candidate instruction uses cur.
397 DEBUG(dbgs() << "Can we DOT Cur Vector MI\n";
399 dbgs() << "in packet\n";);
400 MachineInstr &MJ = *MII;
402 dbgs() << "Checking CUR against ";
405 unsigned DestReg = MI.getOperand(0).getReg();
406 bool FoundMatch = false;
407 for (auto &MO : MJ.operands())
408 if (MO.isReg() && MO.getReg() == DestReg)
413 // Check for existing uses of a vector register within the packet which
414 // would be affected by converting a vector load into .cur formt.
415 for (auto BI : CurrentPacketMIs) {
416 DEBUG(dbgs() << "packet has "; BI->dump(););
417 if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))
421 DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump(););
422 // We can convert the opcode into a .cur.
426 // Promote an instruction to its .new form. At this time, we have already
427 // made a call to canPromoteToDotNew and made sure that it can *indeed* be
429 bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI,
430 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
431 const TargetRegisterClass* RC) {
432 assert (DepType == SDep::Data);
434 if (RC == &Hexagon::PredRegsRegClass)
435 NewOpcode = HII->getDotNewPredOp(MI, MBPI);
437 NewOpcode = HII->getDotNewOp(MI);
438 MI.setDesc(HII->get(NewOpcode));
442 bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) {
443 int NewOpcode = HII->getDotOldOp(MI.getOpcode());
444 MI.setDesc(HII->get(NewOpcode));
448 bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) {
449 unsigned Opc = MI.getOpcode();
451 case Hexagon::S2_storerd_io:
452 case Hexagon::S2_storeri_io:
453 case Hexagon::S2_storerh_io:
454 case Hexagon::S2_storerb_io:
457 llvm_unreachable("Unexpected instruction");
459 unsigned FrameSize = MF.getFrameInfo().getStackSize();
460 MachineOperand &Off = MI.getOperand(1);
461 int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
462 if (HII->isValidOffset(Opc, NewOff)) {
469 void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) {
470 unsigned Opc = MI.getOpcode();
472 case Hexagon::S2_storerd_io:
473 case Hexagon::S2_storeri_io:
474 case Hexagon::S2_storerh_io:
475 case Hexagon::S2_storerb_io:
478 llvm_unreachable("Unexpected instruction");
480 unsigned FrameSize = MF.getFrameInfo().getStackSize();
481 MachineOperand &Off = MI.getOperand(1);
482 Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
491 /// Returns true if an instruction is predicated on p0 and false if it's
492 /// predicated on !p0.
493 static PredicateKind getPredicateSense(const MachineInstr &MI,
494 const HexagonInstrInfo *HII) {
495 if (!HII->isPredicated(MI))
497 if (HII->isPredicatedTrue(MI))
502 static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI,
503 const HexagonInstrInfo *HII) {
504 assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
506 // Post Increment means duplicates. Use dense map to find duplicates in the
507 // list. Caution: Densemap initializes with the minimum of 64 buckets,
508 // whereas there are at most 5 operands in the post increment.
509 DenseSet<unsigned> DefRegsSet;
510 for (auto &MO : MI.operands())
511 if (MO.isReg() && MO.isDef())
512 DefRegsSet.insert(MO.getReg());
514 for (auto &MO : MI.operands())
515 if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
519 const MachineOperand &Op1 = MI.getOperand(1);
520 // The 2nd operand is always the post increment operand in load.
521 assert(Op1.isReg() && "Post increment operand has be to a register.");
524 if (MI.getDesc().mayStore()) {
525 const MachineOperand &Op0 = MI.getOperand(0);
526 // The 1st operand is always the post increment operand in store.
527 assert(Op0.isReg() && "Post increment operand has be to a register.");
531 // we should never come here.
532 llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
535 // Get the value being stored.
536 static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) {
537 // value being stored is always the last operand.
538 return MI.getOperand(MI.getNumOperands()-1);
541 static bool isLoadAbsSet(const MachineInstr &MI) {
542 unsigned Opc = MI.getOpcode();
544 case Hexagon::L4_loadrd_ap:
545 case Hexagon::L4_loadrb_ap:
546 case Hexagon::L4_loadrh_ap:
547 case Hexagon::L4_loadrub_ap:
548 case Hexagon::L4_loadruh_ap:
549 case Hexagon::L4_loadri_ap:
555 static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) {
556 assert(isLoadAbsSet(MI));
557 return MI.getOperand(1);
561 // Can be new value store?
562 // Following restrictions are to be respected in convert a store into
563 // a new value store.
564 // 1. If an instruction uses auto-increment, its address register cannot
565 // be a new-value register. Arch Spec 5.4.2.1
566 // 2. If an instruction uses absolute-set addressing mode, its address
567 // register cannot be a new-value register. Arch Spec 5.4.2.1.
568 // 3. If an instruction produces a 64-bit result, its registers cannot be used
569 // as new-value registers. Arch Spec 5.4.2.2.
570 // 4. If the instruction that sets the new-value register is conditional, then
571 // the instruction that uses the new-value register must also be conditional,
572 // and both must always have their predicates evaluate identically.
573 // Arch Spec 5.4.2.3.
574 // 5. There is an implied restriction that a packet cannot have another store,
575 // if there is a new value store in the packet. Corollary: if there is
576 // already a store in a packet, there can not be a new value store.
577 // Arch Spec: 3.4.4.2
578 bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI,
579 const MachineInstr &PacketMI, unsigned DepReg) {
580 // Make sure we are looking at the store, that can be promoted.
581 if (!HII->mayBeNewStore(MI))
584 // Make sure there is dependency and can be new value'd.
585 const MachineOperand &Val = getStoreValueOperand(MI);
586 if (Val.isReg() && Val.getReg() != DepReg)
589 const MCInstrDesc& MCID = PacketMI.getDesc();
591 // First operand is always the result.
592 const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF);
593 // Double regs can not feed into new value store: PRM section: 5.4.2.2.
594 if (PacketRC == &Hexagon::DoubleRegsRegClass)
597 // New-value stores are of class NV (slot 0), dual stores require class ST
598 // in slot 0 (PRM 5.5).
599 for (auto I : CurrentPacketMIs) {
600 SUnit *PacketSU = MIToSUnit.find(I)->second;
601 if (PacketSU->getInstr()->mayStore())
605 // Make sure it's NOT the post increment register that we are going to
607 if (HII->isPostIncrement(MI) &&
608 getPostIncrementOperand(MI, HII).getReg() == DepReg) {
612 if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() &&
613 getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) {
614 // If source is post_inc, or absolute-set addressing, it can not feed
615 // into new value store
617 // memw(r30 + #-1404) = r2.new -> can not be new value store
618 // arch spec section: 5.4.2.1.
622 if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg)
625 // If the source that feeds the store is predicated, new value store must
626 // also be predicated.
627 if (HII->isPredicated(PacketMI)) {
628 if (!HII->isPredicated(MI))
631 // Check to make sure that they both will have their predicates
632 // evaluate identically.
633 unsigned predRegNumSrc = 0;
634 unsigned predRegNumDst = 0;
635 const TargetRegisterClass* predRegClass = nullptr;
637 // Get predicate register used in the source instruction.
638 for (auto &MO : PacketMI.operands()) {
641 predRegNumSrc = MO.getReg();
642 predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);
643 if (predRegClass == &Hexagon::PredRegsRegClass)
646 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
647 "predicate register not found in a predicated PacketMI instruction");
649 // Get predicate register used in new-value store instruction.
650 for (auto &MO : MI.operands()) {
653 predRegNumDst = MO.getReg();
654 predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);
655 if (predRegClass == &Hexagon::PredRegsRegClass)
658 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
659 "predicate register not found in a predicated MI instruction");
661 // New-value register producer and user (store) need to satisfy these
663 // 1) Both instructions should be predicated on the same register.
664 // 2) If producer of the new-value register is .new predicated then store
665 // should also be .new predicated and if producer is not .new predicated
666 // then store should not be .new predicated.
667 // 3) Both new-value register producer and user should have same predicate
668 // sense, i.e, either both should be negated or both should be non-negated.
669 if (predRegNumDst != predRegNumSrc ||
670 HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||
671 getPredicateSense(MI, HII) != getPredicateSense(PacketMI, HII))
675 // Make sure that other than the new-value register no other store instruction
676 // register has been modified in the same packet. Predicate registers can be
677 // modified by they should not be modified between the producer and the store
678 // instruction as it will make them both conditional on different values.
679 // We already know this to be true for all the instructions before and
680 // including PacketMI. Howerver, we need to perform the check for the
681 // remaining instructions in the packet.
683 unsigned StartCheck = 0;
685 for (auto I : CurrentPacketMIs) {
686 SUnit *TempSU = MIToSUnit.find(I)->second;
687 MachineInstr &TempMI = *TempSU->getInstr();
689 // Following condition is true for all the instructions until PacketMI is
690 // reached (StartCheck is set to 0 before the for loop).
691 // StartCheck flag is 1 for all the instructions after PacketMI.
692 if (&TempMI != &PacketMI && !StartCheck) // Start processing only after
693 continue; // encountering PacketMI.
696 if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence.
699 for (auto &MO : MI.operands())
700 if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI))
704 // Make sure that for non-POST_INC stores:
705 // 1. The only use of reg is DepReg and no other registers.
706 // This handles V4 base+index registers.
707 // The following store can not be dot new.
708 // Eg. r0 = add(r0, #3)
709 // memw(r1+r0<<#2) = r0
710 if (!HII->isPostIncrement(MI)) {
711 for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {
712 const MachineOperand &MO = MI.getOperand(opNum);
713 if (MO.isReg() && MO.getReg() == DepReg)
718 // If data definition is because of implicit definition of the register,
719 // do not newify the store. Eg.
720 // %R9<def> = ZXTH %R12, %D6<imp-use>, %R12<imp-def>
721 // S2_storerh_io %R8, 2, %R12<kill>; mem:ST2[%scevgep343]
722 for (auto &MO : PacketMI.operands()) {
723 if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
725 unsigned R = MO.getReg();
726 if (R == DepReg || HRI->isSuperRegister(DepReg, R))
730 // Handle imp-use of super reg case. There is a target independent side
731 // change that should prevent this situation but I am handling it for
732 // just-in-case. For example, we cannot newify R2 in the following case:
733 // %R3<def> = A2_tfrsi 0;
734 // S2_storeri_io %R0<kill>, 0, %R2<kill>, %D1<imp-use,kill>;
735 for (auto &MO : MI.operands()) {
736 if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
740 // Can be dot new store.
744 // Can this MI to promoted to either new value store or new value jump.
745 bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI,
746 const SUnit *PacketSU, unsigned DepReg,
747 MachineBasicBlock::iterator &MII) {
748 if (!HII->mayBeNewStore(MI))
751 // Check to see the store can be new value'ed.
752 MachineInstr &PacketMI = *PacketSU->getInstr();
753 if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
756 // Check to see the compare/jump can be new value'ed.
757 // This is done as a pass on its own. Don't need to check it here.
761 static bool isImplicitDependency(const MachineInstr &I, unsigned DepReg) {
762 for (auto &MO : I.operands())
763 if (MO.isReg() && MO.isDef() && (MO.getReg() == DepReg) && MO.isImplicit())
768 // Check to see if an instruction can be dot new
769 // There are three kinds.
770 // 1. dot new on predicate - V2/V3/V4
771 // 2. dot new on stores NV/ST - V4
772 // 3. dot new on jump NV/J - V4 -- This is generated in a pass.
773 bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
774 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
775 const TargetRegisterClass* RC) {
776 // Already a dot new instruction.
777 if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
780 if (!isNewifiable(MI, RC))
783 const MachineInstr &PI = *PacketSU->getInstr();
785 // The "new value" cannot come from inline asm.
786 if (PI.isInlineAsm())
789 // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no
791 if (PI.isImplicitDef())
794 // If dependency is trough an implicitly defined register, we should not
796 if (isImplicitDependency(PI, DepReg))
799 const MCInstrDesc& MCID = PI.getDesc();
800 const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF);
801 if (DisableVecDblNVStores && VecRC == &Hexagon::VecDblRegsRegClass)
805 if (RC == &Hexagon::PredRegsRegClass)
806 if (HII->isCondInst(MI) || HII->isJumpR(MI) || MI.isReturn())
807 return HII->predCanBeUsedAsDotNew(PI, DepReg);
809 if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
812 // Create a dot new machine instruction to see if resources can be
813 // allocated. If not, bail out now.
814 int NewOpcode = HII->getDotNewOp(MI);
815 const MCInstrDesc &D = HII->get(NewOpcode);
816 MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc());
817 bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);
818 MF.DeleteMachineInstr(NewMI);
819 if (!ResourcesAvailable)
822 // New Value Store only. New Value Jump generated as a separate pass.
823 if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
829 // Go through the packet instructions and search for an anti dependency between
830 // them and DepReg from MI. Consider this case:
832 // a) %R1<def> = TFRI_cdNotPt %P3, 2
835 // b) %P0<def> = C2_or %P3<kill>, %P0<kill>
836 // c) %P3<def> = C2_tfrrp %R23
837 // d) %R1<def> = C2_cmovenewit %P3, 4
839 // The P3 from a) and d) will be complements after
840 // a)'s P3 is converted to .new form
841 // Anti-dep between c) and b) is irrelevant for this case
842 bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI,
844 SUnit *PacketSUDep = MIToSUnit.find(&MI)->second;
846 for (auto I : CurrentPacketMIs) {
847 // We only care for dependencies to predicated instructions
848 if (!HII->isPredicated(*I))
851 // Scheduling Unit for current insn in the packet
852 SUnit *PacketSU = MIToSUnit.find(I)->second;
854 // Look at dependencies between current members of the packet and
855 // predicate defining instruction MI. Make sure that dependency is
856 // on the exact register we care about.
857 if (PacketSU->isSucc(PacketSUDep)) {
858 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
859 auto &Dep = PacketSU->Succs[i];
860 if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
861 Dep.getReg() == DepReg)
871 /// Gets the predicate register of a predicated instruction.
872 static unsigned getPredicatedRegister(MachineInstr &MI,
873 const HexagonInstrInfo *QII) {
874 /// We use the following rule: The first predicate register that is a use is
875 /// the predicate register of a predicated instruction.
876 assert(QII->isPredicated(MI) && "Must be predicated instruction");
878 for (auto &Op : MI.operands()) {
879 if (Op.isReg() && Op.getReg() && Op.isUse() &&
880 Hexagon::PredRegsRegClass.contains(Op.getReg()))
884 llvm_unreachable("Unknown instruction operand layout");
888 // Given two predicated instructions, this function detects whether
889 // the predicates are complements.
890 bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
892 // If we don't know the predicate sense of the instructions bail out early, we
894 if (getPredicateSense(MI1, HII) == PK_Unknown ||
895 getPredicateSense(MI2, HII) == PK_Unknown)
898 // Scheduling unit for candidate.
899 SUnit *SU = MIToSUnit[&MI1];
901 // One corner case deals with the following scenario:
903 // a) %R24<def> = A2_tfrt %P0, %R25
906 // b) %R25<def> = A2_tfrf %P0, %R24
907 // c) %P0<def> = C2_cmpeqi %R26, 1
910 // On general check a) and b) are complements, but presence of c) will
911 // convert a) to .new form, and then it is not a complement.
912 // We attempt to detect it by analyzing existing dependencies in the packet.
914 // Analyze relationships between all existing members of the packet.
915 // Look for Anti dependecy on the same predicate reg as used in the
917 for (auto I : CurrentPacketMIs) {
918 // Scheduling Unit for current insn in the packet.
919 SUnit *PacketSU = MIToSUnit.find(I)->second;
921 // If this instruction in the packet is succeeded by the candidate...
922 if (PacketSU->isSucc(SU)) {
923 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
924 auto Dep = PacketSU->Succs[i];
925 // The corner case exist when there is true data dependency between
926 // candidate and one of current packet members, this dep is on
927 // predicate reg, and there already exist anti dep on the same pred in
929 if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
930 Hexagon::PredRegsRegClass.contains(Dep.getReg())) {
931 // Here I know that I is predicate setting instruction with true
932 // data dep to candidate on the register we care about - c) in the
933 // above example. Now I need to see if there is an anti dependency
934 // from c) to any other instruction in the same packet on the pred
936 if (restrictingDepExistInPacket(*I, Dep.getReg()))
943 // If the above case does not apply, check regular complement condition.
944 // Check that the predicate register is the same and that the predicate
945 // sense is different We also need to differentiate .old vs. .new: !p0
946 // is not complementary to p0.new.
947 unsigned PReg1 = getPredicatedRegister(MI1, HII);
948 unsigned PReg2 = getPredicatedRegister(MI2, HII);
949 return PReg1 == PReg2 &&
950 Hexagon::PredRegsRegClass.contains(PReg1) &&
951 Hexagon::PredRegsRegClass.contains(PReg2) &&
952 getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) &&
953 HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2);
956 // Initialize packetizer flags.
957 void HexagonPacketizerList::initPacketizerState() {
959 PromotedToDotNew = false;
960 GlueToNewValueJump = false;
961 GlueAllocframeStore = false;
962 FoundSequentialDependence = false;
965 // Ignore bundling of pseudo instructions.
966 bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI,
967 const MachineBasicBlock *) {
968 if (MI.isDebugValue())
971 if (MI.isCFIInstruction())
974 // We must print out inline assembly.
975 if (MI.isInlineAsm())
978 if (MI.isImplicitDef())
981 // We check if MI has any functional units mapped to it. If it doesn't,
982 // we ignore the instruction.
983 const MCInstrDesc& TID = MI.getDesc();
984 auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass());
985 unsigned FuncUnits = IS->getUnits();
989 bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
990 if (MI.isEHLabel() || MI.isCFIInstruction())
993 // Consider inline asm to not be a solo instruction by default.
994 // Inline asm will be put in a packet temporarily, but then it will be
995 // removed, and placed outside of the packet (before or after, depending
996 // on dependencies). This is to reduce the impact of inline asm as a
997 // "packet splitting" instruction.
998 if (MI.isInlineAsm() && !ScheduleInlineAsm)
1001 // From Hexagon V4 Programmer's Reference Manual 3.4.4 Grouping constraints:
1002 // trap, pause, barrier, icinva, isync, and syncht are solo instructions.
1003 // They must not be grouped with other instructions in a packet.
1004 if (isSchedBarrier(MI))
1007 if (HII->isSolo(MI))
1010 if (MI.getOpcode() == Hexagon::A2_nop)
1017 // Quick check if instructions MI and MJ cannot coexist in the same packet.
1018 // Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
1019 // but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
1020 // For full test call this function twice:
1021 // cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
1022 // Doing the test only one way saves the amount of code in this function,
1023 // since every test would need to be repeated with the MI and MJ reversed.
1024 static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ,
1025 const HexagonInstrInfo &HII) {
1026 const MachineFunction *MF = MI.getParent()->getParent();
1027 if (MF->getSubtarget<HexagonSubtarget>().hasV60TOpsOnly() &&
1028 HII.isHVXMemWithAIndirect(MI, MJ))
1031 // An inline asm cannot be together with a branch, because we may not be
1032 // able to remove the asm out after packetizing (i.e. if the asm must be
1033 // moved past the bundle). Similarly, two asms cannot be together to avoid
1034 // complications when determining their relative order outside of a bundle.
1035 if (MI.isInlineAsm())
1036 return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() ||
1037 MJ.isCall() || MJ.isTerminator();
1039 switch (MI.getOpcode()) {
1040 case (Hexagon::S2_storew_locked):
1041 case (Hexagon::S4_stored_locked):
1042 case (Hexagon::L2_loadw_locked):
1043 case (Hexagon::L4_loadd_locked):
1044 case (Hexagon::Y4_l2fetch): {
1045 // These instructions can only be grouped with ALU32 or non-floating-point
1046 // XTYPE instructions. Since there is no convenient way of identifying fp
1047 // XTYPE instructions, only allow grouping with ALU32 for now.
1048 unsigned TJ = HII.getType(MJ);
1049 if (TJ != HexagonII::TypeALU32)
1057 // "False" really means that the quick check failed to determine if
1058 // I and J cannot coexist.
1063 // Full, symmetric check.
1064 bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI,
1065 const MachineInstr &MJ) {
1066 return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII);
1069 void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) {
1070 for (auto &B : MF) {
1071 MachineBasicBlock::iterator BundleIt;
1072 MachineBasicBlock::instr_iterator NextI;
1073 for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) {
1074 NextI = std::next(I);
1075 MachineInstr &MI = *I;
1078 if (!MI.isInsideBundle())
1081 // Decide on where to insert the instruction that we are pulling out.
1082 // Debug instructions always go before the bundle, but the placement of
1083 // INLINE_ASM depends on potential dependencies. By default, try to
1084 // put it before the bundle, but if the asm writes to a register that
1085 // other instructions in the bundle read, then we need to place it
1086 // after the bundle (to preserve the bundle semantics).
1087 bool InsertBeforeBundle;
1088 if (MI.isInlineAsm())
1089 InsertBeforeBundle = !hasWriteToReadDep(MI, *BundleIt, HRI);
1090 else if (MI.isDebugValue())
1091 InsertBeforeBundle = true;
1095 BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle);
1100 // Check if a given instruction is of class "system".
1101 static bool isSystemInstr(const MachineInstr &MI) {
1102 unsigned Opc = MI.getOpcode();
1104 case Hexagon::Y2_barrier:
1105 case Hexagon::Y2_dcfetchbo:
1111 bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I,
1112 const MachineInstr &J) {
1113 // The dependence graph may not include edges between dead definitions,
1114 // so without extra checks, we could end up packetizing two instruction
1115 // defining the same (dead) register.
1116 if (I.isCall() || J.isCall())
1118 if (HII->isPredicated(I) || HII->isPredicated(J))
1121 BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
1122 for (auto &MO : I.operands()) {
1123 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1125 DeadDefs[MO.getReg()] = true;
1128 for (auto &MO : J.operands()) {
1129 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1131 unsigned R = MO.getReg();
1132 if (R != Hexagon::USR_OVF && DeadDefs[R])
1138 bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I,
1139 const MachineInstr &J) {
1140 // A save callee-save register function call can only be in a packet
1141 // with instructions that don't write to the callee-save registers.
1142 if ((HII->isSaveCalleeSavedRegsCall(I) &&
1143 doesModifyCalleeSavedReg(J, HRI)) ||
1144 (HII->isSaveCalleeSavedRegsCall(J) &&
1145 doesModifyCalleeSavedReg(I, HRI)))
1148 // Two control flow instructions cannot go in the same packet.
1149 if (isControlFlow(I) && isControlFlow(J))
1152 // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
1153 // contain a speculative indirect jump,
1154 // a new-value compare jump or a dealloc_return.
1155 auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {
1156 if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
1158 if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))
1163 if (HII->isLoopN(I) && isBadForLoopN(J))
1165 if (HII->isLoopN(J) && isBadForLoopN(I))
1168 // dealloc_return cannot appear in the same packet as a conditional or
1169 // unconditional jump.
1170 return HII->isDeallocRet(I) &&
1171 (J.isBranch() || J.isCall() || J.isBarrier());
1174 bool HexagonPacketizerList::hasV4SpecificDependence(const MachineInstr &I,
1175 const MachineInstr &J) {
1176 bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J);
1177 bool StoreI = I.mayStore(), StoreJ = J.mayStore();
1178 if ((SysI && StoreJ) || (SysJ && StoreI))
1181 if (StoreI && StoreJ) {
1182 if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))
1185 // A memop cannot be in the same packet with another memop or a store.
1186 // Two stores can be together, but here I and J cannot both be stores.
1187 bool MopStI = HII->isMemOp(I) || StoreI;
1188 bool MopStJ = HII->isMemOp(J) || StoreJ;
1189 if (MopStI && MopStJ)
1193 return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));
1196 // SUI is the current instruction that is out side of the current packet.
1197 // SUJ is the current instruction inside the current packet against which that
1198 // SUI will be packetized.
1199 bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
1200 assert(SUI->getInstr() && SUJ->getInstr());
1201 MachineInstr &I = *SUI->getInstr();
1202 MachineInstr &J = *SUJ->getInstr();
1204 // Clear IgnoreDepMIs when Packet starts.
1205 if (CurrentPacketMIs.size() == 1)
1206 IgnoreDepMIs.clear();
1208 MachineBasicBlock::iterator II = I.getIterator();
1210 // Solo instructions cannot go in the packet.
1211 assert(!isSoloInstruction(I) && "Unexpected solo instr!");
1213 if (cannotCoexist(I, J))
1216 Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
1220 // V4 allows dual stores. It does not allow second store, if the first
1221 // store is not in SLOT0. New value store, new value jump, dealloc_return
1222 // and memop always take SLOT0. Arch spec 3.4.4.2.
1223 Dependence = hasV4SpecificDependence(I, J);
1227 // If an instruction feeds new value jump, glue it.
1228 MachineBasicBlock::iterator NextMII = I.getIterator();
1230 if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) {
1231 MachineInstr &NextMI = *NextMII;
1233 bool secondRegMatch = false;
1234 const MachineOperand &NOp0 = NextMI.getOperand(0);
1235 const MachineOperand &NOp1 = NextMI.getOperand(1);
1237 if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg())
1238 secondRegMatch = true;
1240 for (auto T : CurrentPacketMIs) {
1241 SUnit *PacketSU = MIToSUnit.find(T)->second;
1242 MachineInstr &PI = *PacketSU->getInstr();
1243 // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
1249 // 1. Packet does not have a store in it.
1250 // 2. If the first operand of the nvj is newified, and the second
1251 // operand is also a reg, it (second reg) is not defined in
1253 // 3. If the second operand of the nvj is newified, (which means
1254 // first operand is also a reg), first reg is not defined in
1256 if (PI.getOpcode() == Hexagon::S2_allocframe || PI.mayStore() ||
1262 const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
1263 if (OpR.isReg() && PI.modifiesRegister(OpR.getReg(), HRI)) {
1271 GlueToNewValueJump = true;
1274 // There no dependency between a prolog instruction and its successor.
1275 if (!SUJ->isSucc(SUI))
1278 for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
1279 if (FoundSequentialDependence)
1282 if (SUJ->Succs[i].getSUnit() != SUI)
1285 SDep::Kind DepType = SUJ->Succs[i].getKind();
1286 // For direct calls:
1287 // Ignore register dependences for call instructions for packetization
1288 // purposes except for those due to r31 and predicate registers.
1290 // For indirect calls:
1291 // Same as direct calls + check for true dependences to the register
1292 // used in the indirect call.
1294 // We completely ignore Order dependences for call instructions.
1297 // Ignore register dependences for return instructions like jumpr,
1298 // dealloc return unless we have dependencies on the explicit uses
1299 // of the registers used by jumpr (like r31) or dealloc return
1300 // (like r29 or r30).
1301 unsigned DepReg = 0;
1302 const TargetRegisterClass *RC = nullptr;
1303 if (DepType == SDep::Data) {
1304 DepReg = SUJ->Succs[i].getReg();
1305 RC = HRI->getMinimalPhysRegClass(DepReg);
1308 if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) {
1309 if (!isRegDependence(DepType))
1311 if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg()))
1315 if (DepType == SDep::Data) {
1316 if (canPromoteToDotCur(J, SUJ, DepReg, II, RC))
1317 if (promoteToDotCur(J, DepType, II, RC))
1321 // Data dpendence ok if we have load.cur.
1322 if (DepType == SDep::Data && HII->isDotCurInst(J)) {
1323 if (HII->isV60VectorInstruction(I))
1327 // For instructions that can be promoted to dot-new, try to promote.
1328 if (DepType == SDep::Data) {
1329 if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) {
1330 if (promoteToDotNew(I, DepType, II, RC)) {
1331 PromotedToDotNew = true;
1335 if (HII->isNewValueJump(I))
1339 // For predicated instructions, if the predicates are complements then
1340 // there can be no dependence.
1341 if (HII->isPredicated(I) && HII->isPredicated(J) &&
1342 arePredicatesComplements(I, J)) {
1343 // Not always safe to do this translation.
1344 // DAG Builder attempts to reduce dependence edges using transitive
1345 // nature of dependencies. Here is an example:
1347 // r0 = tfr_pt ... (1)
1348 // r0 = tfr_pf ... (2)
1349 // r0 = tfr_pt ... (3)
1351 // There will be an output dependence between (1)->(2) and (2)->(3).
1352 // However, there is no dependence edge between (1)->(3). This results
1353 // in all 3 instructions going in the same packet. We ignore dependce
1354 // only once to avoid this situation.
1355 auto Itr = find(IgnoreDepMIs, &J);
1356 if (Itr != IgnoreDepMIs.end()) {
1360 IgnoreDepMIs.push_back(&I);
1364 // Ignore Order dependences between unconditional direct branches
1365 // and non-control-flow instructions.
1366 if (isDirectJump(I) && !J.isBranch() && !J.isCall() &&
1367 DepType == SDep::Order)
1370 // Ignore all dependences for jumps except for true and output
1372 if (I.isConditionalBranch() && DepType != SDep::Data &&
1373 DepType != SDep::Output)
1376 // Ignore output dependences due to superregs. We can write to two
1377 // different subregisters of R1:0 for instance in the same cycle.
1379 // If neither I nor J defines DepReg, then this is a superfluous output
1380 // dependence. The dependence must be of the form:
1383 // and there is an output dependence between the two instructions with
1385 // We want to ignore these dependences. Ideally, the dependence
1386 // constructor should annotate such dependences. We can then avoid this
1387 // relatively expensive check.
1389 if (DepType == SDep::Output) {
1390 // DepReg is the register that's responsible for the dependence.
1391 unsigned DepReg = SUJ->Succs[i].getReg();
1393 // Check if I and J really defines DepReg.
1394 if (!I.definesRegister(DepReg) && !J.definesRegister(DepReg))
1396 FoundSequentialDependence = true;
1400 // For Order dependences:
1401 // 1. On V4 or later, volatile loads/stores can be packetized together,
1402 // unless other rules prevent is.
1403 // 2. Store followed by a load is not allowed.
1404 // 3. Store followed by a store is only valid on V4 or later.
1405 // 4. Load followed by any memory operation is allowed.
1406 if (DepType == SDep::Order) {
1407 if (!PacketizeVolatiles) {
1408 bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef();
1410 FoundSequentialDependence = true;
1414 // J is first, I is second.
1415 bool LoadJ = J.mayLoad(), StoreJ = J.mayStore();
1416 bool LoadI = I.mayLoad(), StoreI = I.mayStore();
1418 // Two stores are only allowed on V4+. Load following store is never
1421 FoundSequentialDependence = true;
1424 } else if (!LoadJ || (!LoadI && !StoreI)) {
1425 // If J is neither load nor store, assume a dependency.
1426 // If J is a load, but I is neither, also assume a dependency.
1427 FoundSequentialDependence = true;
1430 // Store followed by store: not OK on V2.
1431 // Store followed by load: not OK on all.
1432 // Load followed by store: OK on all.
1433 // Load followed by load: OK on all.
1437 // For V4, special case ALLOCFRAME. Even though there is dependency
1438 // between ALLOCFRAME and subsequent store, allow it to be packetized
1439 // in a same packet. This implies that the store is using the caller's
1440 // SP. Hence, offset needs to be updated accordingly.
1441 if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) {
1442 unsigned Opc = I.getOpcode();
1444 case Hexagon::S2_storerd_io:
1445 case Hexagon::S2_storeri_io:
1446 case Hexagon::S2_storerh_io:
1447 case Hexagon::S2_storerb_io:
1448 if (I.getOperand(0).getReg() == HRI->getStackRegister()) {
1449 // Since this store is to be glued with allocframe in the same
1450 // packet, it will use SP of the previous stack frame, i.e.
1451 // caller's SP. Therefore, we need to recalculate offset
1452 // according to this change.
1453 GlueAllocframeStore = useCallersSP(I);
1454 if (GlueAllocframeStore)
1462 // There are certain anti-dependencies that cannot be ignored.
1464 // J2_call ... %R0<imp-def> ; SUJ
1466 // Those cannot be packetized together, since the call will observe
1467 // the effect of the assignment to R0.
1468 if (DepType == SDep::Anti && J.isCall()) {
1469 // Check if I defines any volatile register. We should also check
1470 // registers that the call may read, but these happen to be a
1471 // subset of the volatile register set.
1472 for (const MCPhysReg *P = J.getDesc().ImplicitDefs; P && *P; ++P) {
1473 if (!I.modifiesRegister(*P, HRI))
1475 FoundSequentialDependence = true;
1480 // Skip over remaining anti-dependences. Two instructions that are
1481 // anti-dependent can share a packet, since in most such cases all
1482 // operands are read before any modifications take place.
1483 // The exceptions are branch and call instructions, since they are
1484 // executed after all other instructions have completed (at least
1486 if (DepType != SDep::Anti) {
1487 FoundSequentialDependence = true;
1492 if (FoundSequentialDependence) {
1500 bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
1501 assert(SUI->getInstr() && SUJ->getInstr());
1502 MachineInstr &I = *SUI->getInstr();
1503 MachineInstr &J = *SUJ->getInstr();
1505 if (cannotCoexist(I, J))
1511 // Check if the instruction was promoted to a dot-new. If so, demote it
1512 // back into a dot-old.
1513 if (PromotedToDotNew)
1517 // Check if the instruction (must be a store) was glued with an allocframe
1518 // instruction. If so, restore its offset to its original value, i.e. use
1519 // current SP instead of caller's SP.
1520 if (GlueAllocframeStore) {
1522 GlueAllocframeStore = false;
1527 MachineBasicBlock::iterator
1528 HexagonPacketizerList::addToPacket(MachineInstr &MI) {
1529 MachineBasicBlock::iterator MII = MI.getIterator();
1530 MachineBasicBlock *MBB = MI.getParent();
1531 if (MI.isImplicitDef()) {
1532 unsigned R = MI.getOperand(0).getReg();
1533 if (Hexagon::IntRegsRegClass.contains(R)) {
1534 MCSuperRegIterator S(R, HRI, false);
1535 MI.addOperand(MachineOperand::CreateReg(*S, true, true));
1539 assert(ResourceTracker->canReserveResources(MI));
1541 bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);
1544 if (GlueToNewValueJump) {
1545 MachineInstr &NvjMI = *++MII;
1546 // We need to put both instructions in the same packet: MI and NvjMI.
1547 // Either of them can require a constant extender. Try to add both to
1548 // the current packet, and if that fails, end the packet and start a
1550 ResourceTracker->reserveResources(MI);
1552 Good = tryAllocateResourcesForConstExt(true);
1554 bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI);
1556 if (ResourceTracker->canReserveResources(NvjMI))
1557 ResourceTracker->reserveResources(NvjMI);
1561 if (Good && ExtNvjMI)
1562 Good = tryAllocateResourcesForConstExt(true);
1566 assert(ResourceTracker->canReserveResources(MI));
1567 ResourceTracker->reserveResources(MI);
1569 assert(canReserveResourcesForConstExt());
1570 tryAllocateResourcesForConstExt(true);
1572 assert(ResourceTracker->canReserveResources(NvjMI));
1573 ResourceTracker->reserveResources(NvjMI);
1575 assert(canReserveResourcesForConstExt());
1576 reserveResourcesForConstExt();
1579 CurrentPacketMIs.push_back(&MI);
1580 CurrentPacketMIs.push_back(&NvjMI);
1584 ResourceTracker->reserveResources(MI);
1585 if (ExtMI && !tryAllocateResourcesForConstExt(true)) {
1587 if (PromotedToDotNew)
1589 if (GlueAllocframeStore) {
1591 GlueAllocframeStore = false;
1593 ResourceTracker->reserveResources(MI);
1594 reserveResourcesForConstExt();
1597 CurrentPacketMIs.push_back(&MI);
1601 void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB,
1602 MachineBasicBlock::iterator MI) {
1603 OldPacketMIs = CurrentPacketMIs;
1604 VLIWPacketizerList::endPacket(MBB, MI);
1607 bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
1608 return !producesStall(MI);
1612 // Return true when ConsMI uses a register defined by ProdMI.
1613 static bool isDependent(const MachineInstr &ProdMI,
1614 const MachineInstr &ConsMI) {
1615 if (!ProdMI.getOperand(0).isReg())
1617 unsigned DstReg = ProdMI.getOperand(0).getReg();
1619 for (auto &Op : ConsMI.operands())
1620 if (Op.isReg() && Op.isUse() && Op.getReg() == DstReg)
1621 // The MIs depend on each other.
1627 // V60 forward scheduling.
1628 bool HexagonPacketizerList::producesStall(const MachineInstr &I) {
1629 // Check whether the previous packet is in a different loop. If this is the
1630 // case, there is little point in trying to avoid a stall because that would
1631 // favor the rare case (loop entry) over the common case (loop iteration).
1633 // TODO: We should really be able to check all the incoming edges if this is
1634 // the first packet in a basic block, so we can avoid stalls from the loop
1636 if (!OldPacketMIs.empty()) {
1637 auto *OldBB = OldPacketMIs.front()->getParent();
1638 auto *ThisBB = I.getParent();
1639 if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB))
1643 // Check for stall between two vector instructions.
1644 if (HII->isV60VectorInstruction(I)) {
1645 for (auto J : OldPacketMIs) {
1646 if (!HII->isV60VectorInstruction(*J))
1648 if (isDependent(*J, I) && !HII->isVecUsableNextPacket(*J, I))
1654 // Check for stall between two scalar instructions. First, check that
1655 // there is no definition of a use in the current packet, because it
1656 // may be a candidate for .new.
1657 for (auto J : CurrentPacketMIs)
1658 if (!HII->isV60VectorInstruction(*J) && isDependent(*J, I))
1661 // Check for stall between I and instructions in the previous packet.
1662 if (MF.getSubtarget<HexagonSubtarget>().useBSBScheduling()) {
1663 for (auto J : OldPacketMIs) {
1664 if (HII->isV60VectorInstruction(*J))
1666 if (!HII->isLateInstrFeedsEarlyInstr(*J, I))
1668 if (isDependent(*J, I) && !HII->canExecuteInBundle(*J, I))
1677 //===----------------------------------------------------------------------===//
1678 // Public Constructor Functions
1679 //===----------------------------------------------------------------------===//
1681 FunctionPass *llvm::createHexagonPacketizer() {
1682 return new HexagonPacketizer();