1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
11 //===----------------------------------------------------------------------===//
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/CodeGen/LiveIntervals.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineMemOperand.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/PseudoSourceValue.h"
30 #include "llvm/CodeGen/ScheduleDAG.h"
31 #include "llvm/CodeGen/SlotIndexes.h"
32 #include "llvm/CodeGen/StackMaps.h"
33 #include "llvm/MC/MCAsmInfo.h"
34 #include "llvm/MC/MCInst.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/TargetRegistry.h"
39 #include "llvm/Support/raw_ostream.h"
43 #define DEBUG_TYPE "ppc-instr-info"
45 #define GET_INSTRMAP_INFO
46 #define GET_INSTRINFO_CTOR_DTOR
47 #include "PPCGenInstrInfo.inc"
49 STATISTIC(NumStoreSPILLVSRRCAsVec,
50 "Number of spillvsrrc spilled to stack as vec");
51 STATISTIC(NumStoreSPILLVSRRCAsGpr,
52 "Number of spillvsrrc spilled to stack as gpr");
53 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
54 STATISTIC(CmpIselsConverted,
55 "Number of ISELs that depend on comparison of constants converted");
56 STATISTIC(MissedConvertibleImmediateInstrs,
57 "Number of compare-immediate instructions fed by constants");
58 STATISTIC(NumRcRotatesConvertedToRcAnd,
59 "Number of record-form rotates converted to record-form andi");
62 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
63 cl::desc("Disable analysis for CTR loops"));
65 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
66 cl::desc("Disable compare instruction optimization"), cl::Hidden);
68 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
69 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
73 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
74 cl::desc("Use the old (incorrect) instruction latency calculation"));
76 // Pin the vtable to this file.
77 void PPCInstrInfo::anchor() {}
79 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
80 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
81 /* CatchRetOpcode */ -1,
82 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
83 Subtarget(STI), RI(STI.getTargetMachine()) {}
85 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
86 /// this target when scheduling the DAG.
87 ScheduleHazardRecognizer *
88 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
89 const ScheduleDAG *DAG) const {
91 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
92 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
93 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
94 const InstrItineraryData *II =
95 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
96 return new ScoreboardHazardRecognizer(II, DAG);
99 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
102 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
103 /// to use for this target when scheduling the DAG.
104 ScheduleHazardRecognizer *
105 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
106 const ScheduleDAG *DAG) const {
108 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
110 // FIXME: Leaving this as-is until we have POWER9 scheduling info
111 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
112 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
114 // Most subtargets use a PPC970 recognizer.
115 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
116 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
117 assert(DAG->TII && "No InstrInfo?");
119 return new PPCHazardRecognizer970(*DAG);
122 return new ScoreboardHazardRecognizer(II, DAG);
125 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
126 const MachineInstr &MI,
127 unsigned *PredCost) const {
128 if (!ItinData || UseOldLatencyCalc)
129 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
131 // The default implementation of getInstrLatency calls getStageLatency, but
132 // getStageLatency does not do the right thing for us. While we have
133 // itinerary, most cores are fully pipelined, and so the itineraries only
134 // express the first part of the pipeline, not every stage. Instead, we need
135 // to use the listed output operand cycle number (using operand 0 here, which
138 unsigned Latency = 1;
139 unsigned DefClass = MI.getDesc().getSchedClass();
140 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
141 const MachineOperand &MO = MI.getOperand(i);
142 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
145 int Cycle = ItinData->getOperandCycle(DefClass, i);
149 Latency = std::max(Latency, (unsigned) Cycle);
155 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
156 const MachineInstr &DefMI, unsigned DefIdx,
157 const MachineInstr &UseMI,
158 unsigned UseIdx) const {
159 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
162 if (!DefMI.getParent())
165 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
166 Register Reg = DefMO.getReg();
169 if (Register::isVirtualRegister(Reg)) {
170 const MachineRegisterInfo *MRI =
171 &DefMI.getParent()->getParent()->getRegInfo();
172 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
173 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
175 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
176 PPC::CRBITRCRegClass.contains(Reg);
179 if (UseMI.isBranch() && IsRegCR) {
181 Latency = getInstrLatency(ItinData, DefMI);
183 // On some cores, there is an additional delay between writing to a condition
184 // register, and using it from a branch.
185 unsigned Directive = Subtarget.getCPUDirective();
199 // FIXME: Is this needed for POWER9?
208 /// This is an architecture-specific helper function of reassociateOps.
209 /// Set special operand attributes for new instructions after reassociation.
210 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
211 MachineInstr &OldMI2,
212 MachineInstr &NewMI1,
213 MachineInstr &NewMI2) const {
214 // Propagate FP flags from the original instructions.
215 // But clear poison-generating flags because those may not be valid now.
216 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
217 NewMI1.setFlags(IntersectedFlags);
218 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
219 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
220 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact);
222 NewMI2.setFlags(IntersectedFlags);
223 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap);
224 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap);
225 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact);
228 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
229 uint16_t Flags) const {
231 MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
232 MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
233 MI.clearFlag(MachineInstr::MIFlag::IsExact);
236 // This function does not list all associative and commutative operations, but
237 // only those worth feeding through the machine combiner in an attempt to
238 // reduce the critical path. Mostly, this means floating-point operations,
239 // because they have high latencies(>=5) (compared to other operations, such as
240 // and/or, which are also associative and commutative, but have low latencies).
241 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
242 switch (Inst.getOpcode()) {
270 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
271 Inst.getFlag(MachineInstr::MIFlag::FmNsz);
284 #define InfoArrayIdxFMAInst 0
285 #define InfoArrayIdxFAddInst 1
286 #define InfoArrayIdxFMULInst 2
287 #define InfoArrayIdxAddOpIdx 3
288 #define InfoArrayIdxMULOpIdx 4
289 // Array keeps info for FMA instructions:
290 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
291 // Index 1(InfoArrayIdxFAddInst): ADD instruction assoaicted with FMA;
292 // Index 2(InfoArrayIdxFMULInst): MUL instruction assoaicted with FMA;
293 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
294 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
295 // second MUL operand index is plus 1.
296 static const uint16_t FMAOpIdxInfo[][5] = {
297 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
298 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2},
299 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2},
300 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
301 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2},
302 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1},
303 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1},
304 {PPC::QVFMADDSs, PPC::QVFADDSs, PPC::QVFMULSs, 3, 1},
305 {PPC::QVFMADD, PPC::QVFADD, PPC::QVFMUL, 3, 1}};
307 // Check if an opcode is a FMA instruction. If it is, return the index in array
308 // FMAOpIdxInfo. Otherwise, return -1.
309 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
310 for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++)
311 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
316 // Try to reassociate FMA chains like below:
319 // A = FADD X, Y (Leaf)
320 // B = FMA A, M21, M22 (Prev)
321 // C = FMA B, M31, M32 (Root)
323 // A = FMA X, M21, M22
324 // B = FMA Y, M31, M32
328 // A = FMA X, M11, M12 (Leaf)
329 // B = FMA A, M21, M22 (Prev)
330 // C = FMA B, M31, M32 (Root)
333 // B = FMA X, M21, M22
334 // D = FMA A, M31, M32
337 // breaking the dependency between A and B, allowing FMA to be executed in
338 // parallel (or back-to-back in a pipeline) instead of depending on each other.
339 bool PPCInstrInfo::getFMAPatterns(
341 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
342 MachineBasicBlock *MBB = Root.getParent();
343 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
345 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
346 for (const auto &MO : Instr.explicit_operands())
347 if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg())))
352 auto IsReassociable = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
353 bool IsLeaf, bool IsAdd) {
356 Idx = getFMAOpIdxInfo(Instr.getOpcode());
359 } else if (Instr.getOpcode() !=
360 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())]
361 [InfoArrayIdxFAddInst])
364 // Instruction can be reassociated.
365 // fast math flags may prohibit reassociation.
366 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
367 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
370 // Instruction operands are virtual registers for reassociation.
371 if (!IsAllOpsVirtualReg(Instr))
377 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
379 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
380 MachineInstr *MIAdd = MRI.getUniqueVRegDef(OpAdd.getReg());
381 // If 'add' operand's def is not in current block, don't do ILP related opt.
382 if (!MIAdd || MIAdd->getParent() != MBB)
385 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
386 // as this fma will be changed later.
387 return IsLeaf ? true : MRI.hasOneNonDBGUse(OpAdd.getReg());
390 int16_t AddOpIdx = -1;
391 // Root must be a valid FMA like instruction.
392 if (!IsReassociable(Root, AddOpIdx, false, false))
395 assert((AddOpIdx >= 0) && "add operand index not right!");
397 Register RegB = Root.getOperand(AddOpIdx).getReg();
398 MachineInstr *Prev = MRI.getUniqueVRegDef(RegB);
400 // Prev must be a valid FMA like instruction.
402 if (!IsReassociable(*Prev, AddOpIdx, false, false))
405 assert((AddOpIdx >= 0) && "add operand index not right!");
407 Register RegA = Prev->getOperand(AddOpIdx).getReg();
408 MachineInstr *Leaf = MRI.getUniqueVRegDef(RegA);
410 if (IsReassociable(*Leaf, AddOpIdx, true, false)) {
411 Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
414 if (IsReassociable(*Leaf, AddOpIdx, true, true)) {
415 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
421 bool PPCInstrInfo::getMachineCombinerPatterns(
423 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
424 // Using the machine combiner in this way is potentially expensive, so
425 // restrict to when aggressive optimizations are desired.
426 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
429 if (getFMAPatterns(Root, Patterns))
432 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
435 void PPCInstrInfo::genAlternativeCodeSequence(
436 MachineInstr &Root, MachineCombinerPattern Pattern,
437 SmallVectorImpl<MachineInstr *> &InsInstrs,
438 SmallVectorImpl<MachineInstr *> &DelInstrs,
439 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
441 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
442 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
443 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
446 // Reassociate default patterns.
447 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
448 DelInstrs, InstrIdxForVirtReg);
453 // Currently, only handle two patterns REASSOC_XY_AMM_BMM and
454 // REASSOC_XMM_AMM_BMM. See comments for getFMAPatterns.
455 void PPCInstrInfo::reassociateFMA(
456 MachineInstr &Root, MachineCombinerPattern Pattern,
457 SmallVectorImpl<MachineInstr *> &InsInstrs,
458 SmallVectorImpl<MachineInstr *> &DelInstrs,
459 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
460 MachineFunction *MF = Root.getMF();
461 MachineRegisterInfo &MRI = MF->getRegInfo();
462 MachineOperand &OpC = Root.getOperand(0);
463 Register RegC = OpC.getReg();
464 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
465 MRI.constrainRegClass(RegC, RC);
467 unsigned FmaOp = Root.getOpcode();
468 int16_t Idx = getFMAOpIdxInfo(FmaOp);
469 assert(Idx >= 0 && "Root must be a FMA instruction");
471 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
472 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
473 MachineInstr *Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
475 MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
476 uint16_t IntersectedFlags =
477 Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
479 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
481 Reg = Operand.getReg();
482 MRI.constrainRegClass(Reg, RC);
483 KillFlag = Operand.isKill();
486 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
487 Register &MulOp2, bool &MulOp1KillFlag,
488 bool &MulOp2KillFlag) {
489 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
490 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
493 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32;
494 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
495 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false;
497 GetFMAInstrInfo(Root, RegM31, RegM32, KillM31, KillM32);
498 GetFMAInstrInfo(*Prev, RegM21, RegM22, KillM21, KillM22);
500 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
501 GetFMAInstrInfo(*Leaf, RegM11, RegM12, KillM11, KillM12);
502 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
503 } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
504 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
505 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
508 // Create new virtual registers for the new results instead of
509 // recycling legacy ones because the MachineCombiner's computation of the
510 // critical path requires a new register definition rather than an existing
512 Register NewVRA = MRI.createVirtualRegister(RC);
513 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
515 Register NewVRB = MRI.createVirtualRegister(RC);
516 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
519 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
520 NewVRD = MRI.createVirtualRegister(RC);
521 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
524 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
525 Register RegMul1, bool KillRegMul1,
526 Register RegMul2, bool KillRegMul2) {
527 MI->getOperand(AddOpIdx).setReg(RegAdd);
528 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
529 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
530 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
531 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
532 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
535 if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
536 // Create new instructions for insertion.
537 MachineInstrBuilder MINewB =
538 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
539 .addReg(RegX, getKillRegState(KillX))
540 .addReg(RegM21, getKillRegState(KillM21))
541 .addReg(RegM22, getKillRegState(KillM22));
542 MachineInstrBuilder MINewA =
543 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
544 .addReg(RegY, getKillRegState(KillY))
545 .addReg(RegM31, getKillRegState(KillM31))
546 .addReg(RegM32, getKillRegState(KillM32));
547 // If AddOpIdx is not 1, adjust the order.
549 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
550 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
553 MachineInstrBuilder MINewC =
554 BuildMI(*MF, Root.getDebugLoc(),
555 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
556 .addReg(NewVRB, getKillRegState(true))
557 .addReg(NewVRA, getKillRegState(true));
559 // Update flags for newly created instructions.
560 setSpecialOperandAttr(*MINewA, IntersectedFlags);
561 setSpecialOperandAttr(*MINewB, IntersectedFlags);
562 setSpecialOperandAttr(*MINewC, IntersectedFlags);
564 // Record new instructions for insertion.
565 InsInstrs.push_back(MINewA);
566 InsInstrs.push_back(MINewB);
567 InsInstrs.push_back(MINewC);
568 } else if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
569 assert(NewVRD && "new FMA register not created!");
570 // Create new instructions for insertion.
571 MachineInstrBuilder MINewA =
572 BuildMI(*MF, Leaf->getDebugLoc(),
573 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
574 .addReg(RegM11, getKillRegState(KillM11))
575 .addReg(RegM12, getKillRegState(KillM12));
576 MachineInstrBuilder MINewB =
577 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
578 .addReg(RegX, getKillRegState(KillX))
579 .addReg(RegM21, getKillRegState(KillM21))
580 .addReg(RegM22, getKillRegState(KillM22));
581 MachineInstrBuilder MINewD =
582 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
583 .addReg(NewVRA, getKillRegState(true))
584 .addReg(RegM31, getKillRegState(KillM31))
585 .addReg(RegM32, getKillRegState(KillM32));
586 // If AddOpIdx is not 1, adjust the order.
588 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
589 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
593 MachineInstrBuilder MINewC =
594 BuildMI(*MF, Root.getDebugLoc(),
595 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
596 .addReg(NewVRB, getKillRegState(true))
597 .addReg(NewVRD, getKillRegState(true));
599 // Update flags for newly created instructions.
600 setSpecialOperandAttr(*MINewA, IntersectedFlags);
601 setSpecialOperandAttr(*MINewB, IntersectedFlags);
602 setSpecialOperandAttr(*MINewD, IntersectedFlags);
603 setSpecialOperandAttr(*MINewC, IntersectedFlags);
605 // Record new instructions for insertion.
606 InsInstrs.push_back(MINewA);
607 InsInstrs.push_back(MINewB);
608 InsInstrs.push_back(MINewD);
609 InsInstrs.push_back(MINewC);
612 assert(!InsInstrs.empty() &&
613 "Insertion instructions set should not be empty!");
615 // Record old instructions for deletion.
616 DelInstrs.push_back(Leaf);
617 DelInstrs.push_back(Prev);
618 DelInstrs.push_back(&Root);
621 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
622 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
623 Register &SrcReg, Register &DstReg,
624 unsigned &SubIdx) const {
625 switch (MI.getOpcode()) {
626 default: return false;
629 case PPC::EXTSW_32_64:
630 SrcReg = MI.getOperand(1).getReg();
631 DstReg = MI.getOperand(0).getReg();
632 SubIdx = PPC::sub_32;
637 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
638 int &FrameIndex) const {
639 unsigned Opcode = MI.getOpcode();
640 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
641 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
643 if (End != std::find(OpcodesForSpill, End, Opcode)) {
644 // Check for the operands added by addFrameReference (the immediate is the
645 // offset which defaults to 0).
646 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
647 MI.getOperand(2).isFI()) {
648 FrameIndex = MI.getOperand(2).getIndex();
649 return MI.getOperand(0).getReg();
655 // For opcodes with the ReMaterializable flag set, this function is called to
656 // verify the instruction is really rematable.
657 bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
658 AliasAnalysis *AA) const {
659 switch (MI.getOpcode()) {
661 // This function should only be called for opcodes with the ReMaterializable
663 llvm_unreachable("Unknown rematerializable operation!");
670 case PPC::ADDIStocHA:
671 case PPC::ADDIStocHA8:
673 case PPC::LOAD_STACK_GUARD:
677 case PPC::XXLEQVOnes:
681 case PPC::V_SETALLONESB:
682 case PPC::V_SETALLONESH:
683 case PPC::V_SETALLONES:
691 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
692 int &FrameIndex) const {
693 unsigned Opcode = MI.getOpcode();
694 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
695 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
697 if (End != std::find(OpcodesForSpill, End, Opcode)) {
698 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
699 MI.getOperand(2).isFI()) {
700 FrameIndex = MI.getOperand(2).getIndex();
701 return MI.getOperand(0).getReg();
707 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
709 unsigned OpIdx2) const {
710 MachineFunction &MF = *MI.getParent()->getParent();
712 // Normal instructions can be commuted the obvious way.
713 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
714 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
715 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
716 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
717 // changing the relative order of the mask operands might change what happens
718 // to the high-bits of the mask (and, thus, the result).
720 // Cannot commute if it has a non-zero rotate count.
721 if (MI.getOperand(3).getImm() != 0)
724 // If we have a zero rotate count, we have:
726 // Op0 = (Op1 & ~M) | (Op2 & M)
728 // M = mask((ME+1)&31, (MB-1)&31)
729 // Op0 = (Op2 & ~M) | (Op1 & M)
732 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
733 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
734 Register Reg0 = MI.getOperand(0).getReg();
735 Register Reg1 = MI.getOperand(1).getReg();
736 Register Reg2 = MI.getOperand(2).getReg();
737 unsigned SubReg1 = MI.getOperand(1).getSubReg();
738 unsigned SubReg2 = MI.getOperand(2).getSubReg();
739 bool Reg1IsKill = MI.getOperand(1).isKill();
740 bool Reg2IsKill = MI.getOperand(2).isKill();
741 bool ChangeReg0 = false;
742 // If machine instrs are no longer in two-address forms, update
743 // destination register as well.
745 // Must be two address instruction!
746 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
747 "Expecting a two-address instruction!");
748 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
754 unsigned MB = MI.getOperand(4).getImm();
755 unsigned ME = MI.getOperand(5).getImm();
757 // We can't commute a trivial mask (there is no way to represent an all-zero
759 if (MB == 0 && ME == 31)
763 // Create a new instruction.
764 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
765 bool Reg0IsDead = MI.getOperand(0).isDead();
766 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
767 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
768 .addReg(Reg2, getKillRegState(Reg2IsKill))
769 .addReg(Reg1, getKillRegState(Reg1IsKill))
770 .addImm((ME + 1) & 31)
771 .addImm((MB - 1) & 31);
775 MI.getOperand(0).setReg(Reg2);
776 MI.getOperand(0).setSubReg(SubReg2);
778 MI.getOperand(2).setReg(Reg1);
779 MI.getOperand(1).setReg(Reg2);
780 MI.getOperand(2).setSubReg(SubReg1);
781 MI.getOperand(1).setSubReg(SubReg2);
782 MI.getOperand(2).setIsKill(Reg1IsKill);
783 MI.getOperand(1).setIsKill(Reg2IsKill);
785 // Swap the mask around.
786 MI.getOperand(4).setImm((ME + 1) & 31);
787 MI.getOperand(5).setImm((MB - 1) & 31);
791 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
793 unsigned &SrcOpIdx2) const {
794 // For VSX A-Type FMA instructions, it is the first two operands that can be
795 // commuted, however, because the non-encoded tied input operand is listed
796 // first, the operands to swap are actually the second and third.
798 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
800 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
802 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
804 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
807 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
808 MachineBasicBlock::iterator MI) const {
809 // This function is used for scheduling, and the nop wanted here is the type
810 // that terminates dispatch groups on the POWER cores.
811 unsigned Directive = Subtarget.getCPUDirective();
814 default: Opcode = PPC::NOP; break;
815 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
816 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
817 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
818 // FIXME: Update when POWER9 scheduling model is ready.
819 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
823 BuildMI(MBB, MI, DL, get(Opcode));
826 /// Return the noop instruction to use for a noop.
827 void PPCInstrInfo::getNoop(MCInst &NopInst) const {
828 NopInst.setOpcode(PPC::NOP);
832 // Note: If the condition register is set to CTR or CTR8 then this is a
833 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
834 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
835 MachineBasicBlock *&TBB,
836 MachineBasicBlock *&FBB,
837 SmallVectorImpl<MachineOperand> &Cond,
838 bool AllowModify) const {
839 bool isPPC64 = Subtarget.isPPC64();
841 // If the block has no terminators, it just falls into the block after it.
842 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
846 if (!isUnpredicatedTerminator(*I))
850 // If the BB ends with an unconditional branch to the fallthrough BB,
851 // we eliminate the branch instruction.
852 if (I->getOpcode() == PPC::B &&
853 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
854 I->eraseFromParent();
856 // We update iterator after deleting the last branch.
857 I = MBB.getLastNonDebugInstr();
858 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
863 // Get the last instruction in the block.
864 MachineInstr &LastInst = *I;
866 // If there is only one terminator instruction, process it.
867 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
868 if (LastInst.getOpcode() == PPC::B) {
869 if (!LastInst.getOperand(0).isMBB())
871 TBB = LastInst.getOperand(0).getMBB();
873 } else if (LastInst.getOpcode() == PPC::BCC) {
874 if (!LastInst.getOperand(2).isMBB())
876 // Block ends with fall-through condbranch.
877 TBB = LastInst.getOperand(2).getMBB();
878 Cond.push_back(LastInst.getOperand(0));
879 Cond.push_back(LastInst.getOperand(1));
881 } else if (LastInst.getOpcode() == PPC::BC) {
882 if (!LastInst.getOperand(1).isMBB())
884 // Block ends with fall-through condbranch.
885 TBB = LastInst.getOperand(1).getMBB();
886 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
887 Cond.push_back(LastInst.getOperand(0));
889 } else if (LastInst.getOpcode() == PPC::BCn) {
890 if (!LastInst.getOperand(1).isMBB())
892 // Block ends with fall-through condbranch.
893 TBB = LastInst.getOperand(1).getMBB();
894 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
895 Cond.push_back(LastInst.getOperand(0));
897 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
898 LastInst.getOpcode() == PPC::BDNZ) {
899 if (!LastInst.getOperand(0).isMBB())
901 if (DisableCTRLoopAnal)
903 TBB = LastInst.getOperand(0).getMBB();
904 Cond.push_back(MachineOperand::CreateImm(1));
905 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
908 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
909 LastInst.getOpcode() == PPC::BDZ) {
910 if (!LastInst.getOperand(0).isMBB())
912 if (DisableCTRLoopAnal)
914 TBB = LastInst.getOperand(0).getMBB();
915 Cond.push_back(MachineOperand::CreateImm(0));
916 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
921 // Otherwise, don't know what this is.
925 // Get the instruction before it if it's a terminator.
926 MachineInstr &SecondLastInst = *I;
928 // If there are three terminators, we don't know what sort of block this is.
929 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
932 // If the block ends with PPC::B and PPC:BCC, handle it.
933 if (SecondLastInst.getOpcode() == PPC::BCC &&
934 LastInst.getOpcode() == PPC::B) {
935 if (!SecondLastInst.getOperand(2).isMBB() ||
936 !LastInst.getOperand(0).isMBB())
938 TBB = SecondLastInst.getOperand(2).getMBB();
939 Cond.push_back(SecondLastInst.getOperand(0));
940 Cond.push_back(SecondLastInst.getOperand(1));
941 FBB = LastInst.getOperand(0).getMBB();
943 } else if (SecondLastInst.getOpcode() == PPC::BC &&
944 LastInst.getOpcode() == PPC::B) {
945 if (!SecondLastInst.getOperand(1).isMBB() ||
946 !LastInst.getOperand(0).isMBB())
948 TBB = SecondLastInst.getOperand(1).getMBB();
949 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
950 Cond.push_back(SecondLastInst.getOperand(0));
951 FBB = LastInst.getOperand(0).getMBB();
953 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
954 LastInst.getOpcode() == PPC::B) {
955 if (!SecondLastInst.getOperand(1).isMBB() ||
956 !LastInst.getOperand(0).isMBB())
958 TBB = SecondLastInst.getOperand(1).getMBB();
959 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
960 Cond.push_back(SecondLastInst.getOperand(0));
961 FBB = LastInst.getOperand(0).getMBB();
963 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
964 SecondLastInst.getOpcode() == PPC::BDNZ) &&
965 LastInst.getOpcode() == PPC::B) {
966 if (!SecondLastInst.getOperand(0).isMBB() ||
967 !LastInst.getOperand(0).isMBB())
969 if (DisableCTRLoopAnal)
971 TBB = SecondLastInst.getOperand(0).getMBB();
972 Cond.push_back(MachineOperand::CreateImm(1));
973 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
975 FBB = LastInst.getOperand(0).getMBB();
977 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
978 SecondLastInst.getOpcode() == PPC::BDZ) &&
979 LastInst.getOpcode() == PPC::B) {
980 if (!SecondLastInst.getOperand(0).isMBB() ||
981 !LastInst.getOperand(0).isMBB())
983 if (DisableCTRLoopAnal)
985 TBB = SecondLastInst.getOperand(0).getMBB();
986 Cond.push_back(MachineOperand::CreateImm(0));
987 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
989 FBB = LastInst.getOperand(0).getMBB();
993 // If the block ends with two PPC:Bs, handle it. The second one is not
994 // executed, so remove it.
995 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
996 if (!SecondLastInst.getOperand(0).isMBB())
998 TBB = SecondLastInst.getOperand(0).getMBB();
1001 I->eraseFromParent();
1005 // Otherwise, can't handle this.
1009 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1010 int *BytesRemoved) const {
1011 assert(!BytesRemoved && "code size not handled");
1013 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1017 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1018 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1019 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1020 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1023 // Remove the branch.
1024 I->eraseFromParent();
1028 if (I == MBB.begin()) return 1;
1030 if (I->getOpcode() != PPC::BCC &&
1031 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1032 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1033 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1036 // Remove the branch.
1037 I->eraseFromParent();
1041 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1042 MachineBasicBlock *TBB,
1043 MachineBasicBlock *FBB,
1044 ArrayRef<MachineOperand> Cond,
1046 int *BytesAdded) const {
1047 // Shouldn't be a fall through.
1048 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1049 assert((Cond.size() == 2 || Cond.size() == 0) &&
1050 "PPC branch conditions have two components!");
1051 assert(!BytesAdded && "code size not handled");
1053 bool isPPC64 = Subtarget.isPPC64();
1057 if (Cond.empty()) // Unconditional branch
1058 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1059 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1060 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1061 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1062 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1063 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1064 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1065 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1066 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1067 else // Conditional branch
1068 BuildMI(&MBB, DL, get(PPC::BCC))
1069 .addImm(Cond[0].getImm())
1075 // Two-way Conditional Branch.
1076 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1077 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1078 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1079 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1080 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1081 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1082 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1083 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1085 BuildMI(&MBB, DL, get(PPC::BCC))
1086 .addImm(Cond[0].getImm())
1089 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1094 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1095 ArrayRef<MachineOperand> Cond,
1096 Register DstReg, Register TrueReg,
1097 Register FalseReg, int &CondCycles,
1098 int &TrueCycles, int &FalseCycles) const {
1099 if (Cond.size() != 2)
1102 // If this is really a bdnz-like condition, then it cannot be turned into a
1104 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1107 // Check register classes.
1108 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1109 const TargetRegisterClass *RC =
1110 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1114 // isel is for regular integer GPRs only.
1115 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1116 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1117 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1118 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1121 // FIXME: These numbers are for the A2, how well they work for other cores is
1122 // an open question. On the A2, the isel instruction has a 2-cycle latency
1123 // but single-cycle throughput. These numbers are used in combination with
1124 // the MispredictPenalty setting from the active SchedMachineModel.
1132 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1133 MachineBasicBlock::iterator MI,
1134 const DebugLoc &dl, Register DestReg,
1135 ArrayRef<MachineOperand> Cond, Register TrueReg,
1136 Register FalseReg) const {
1137 assert(Cond.size() == 2 &&
1138 "PPC branch conditions have two components!");
1140 // Get the register classes.
1141 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1142 const TargetRegisterClass *RC =
1143 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1144 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1146 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1147 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1149 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1150 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1151 "isel is for regular integer GPRs only");
1153 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1154 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1156 unsigned SubIdx = 0;
1157 bool SwapOps = false;
1158 switch (SelectPred) {
1160 case PPC::PRED_EQ_MINUS:
1161 case PPC::PRED_EQ_PLUS:
1162 SubIdx = PPC::sub_eq; SwapOps = false; break;
1164 case PPC::PRED_NE_MINUS:
1165 case PPC::PRED_NE_PLUS:
1166 SubIdx = PPC::sub_eq; SwapOps = true; break;
1168 case PPC::PRED_LT_MINUS:
1169 case PPC::PRED_LT_PLUS:
1170 SubIdx = PPC::sub_lt; SwapOps = false; break;
1172 case PPC::PRED_GE_MINUS:
1173 case PPC::PRED_GE_PLUS:
1174 SubIdx = PPC::sub_lt; SwapOps = true; break;
1176 case PPC::PRED_GT_MINUS:
1177 case PPC::PRED_GT_PLUS:
1178 SubIdx = PPC::sub_gt; SwapOps = false; break;
1180 case PPC::PRED_LE_MINUS:
1181 case PPC::PRED_LE_PLUS:
1182 SubIdx = PPC::sub_gt; SwapOps = true; break;
1184 case PPC::PRED_UN_MINUS:
1185 case PPC::PRED_UN_PLUS:
1186 SubIdx = PPC::sub_un; SwapOps = false; break;
1188 case PPC::PRED_NU_MINUS:
1189 case PPC::PRED_NU_PLUS:
1190 SubIdx = PPC::sub_un; SwapOps = true; break;
1191 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1192 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1195 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1196 SecondReg = SwapOps ? TrueReg : FalseReg;
1198 // The first input register of isel cannot be r0. If it is a member
1199 // of a register class that can be r0, then copy it first (the
1200 // register allocator should eliminate the copy).
1201 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1202 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1203 const TargetRegisterClass *FirstRC =
1204 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1205 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1206 Register OldFirstReg = FirstReg;
1207 FirstReg = MRI.createVirtualRegister(FirstRC);
1208 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1209 .addReg(OldFirstReg);
1212 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1213 .addReg(FirstReg).addReg(SecondReg)
1214 .addReg(Cond[1].getReg(), 0, SubIdx);
1217 static unsigned getCRBitValue(unsigned CRBit) {
1219 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1220 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1221 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1222 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1224 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1225 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1226 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1227 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1229 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1230 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1231 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1232 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1234 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1235 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1236 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1237 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1240 assert(Ret != 4 && "Invalid CR bit register");
1244 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1245 MachineBasicBlock::iterator I,
1246 const DebugLoc &DL, MCRegister DestReg,
1247 MCRegister SrcReg, bool KillSrc) const {
1248 // We can end up with self copies and similar things as a result of VSX copy
1249 // legalization. Promote them here.
1250 const TargetRegisterInfo *TRI = &getRegisterInfo();
1251 if (PPC::F8RCRegClass.contains(DestReg) &&
1252 PPC::VSRCRegClass.contains(SrcReg)) {
1253 MCRegister SuperReg =
1254 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1256 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1257 llvm_unreachable("nop VSX copy");
1260 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1261 PPC::VSRCRegClass.contains(DestReg)) {
1262 MCRegister SuperReg =
1263 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1265 if (VSXSelfCopyCrash && DestReg == SuperReg)
1266 llvm_unreachable("nop VSX copy");
1271 // Different class register copy
1272 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1273 PPC::GPRCRegClass.contains(DestReg)) {
1274 MCRegister CRReg = getCRFromCRBit(SrcReg);
1275 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1276 getKillRegState(KillSrc);
1277 // Rotate the CR bit in the CR fields to be the least significant bit and
1278 // then mask with 0x1 (MB = ME = 31).
1279 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1280 .addReg(DestReg, RegState::Kill)
1281 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1285 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1286 PPC::G8RCRegClass.contains(DestReg)) {
1287 BuildMI(MBB, I, DL, get(PPC::MFOCRF8), DestReg).addReg(SrcReg);
1288 getKillRegState(KillSrc);
1290 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1291 PPC::GPRCRegClass.contains(DestReg)) {
1292 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(SrcReg);
1293 getKillRegState(KillSrc);
1295 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1296 PPC::VSFRCRegClass.contains(DestReg)) {
1297 assert(Subtarget.hasDirectMove() &&
1298 "Subtarget doesn't support directmove, don't know how to copy.");
1299 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1301 getKillRegState(KillSrc);
1303 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1304 PPC::G8RCRegClass.contains(DestReg)) {
1305 assert(Subtarget.hasDirectMove() &&
1306 "Subtarget doesn't support directmove, don't know how to copy.");
1307 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1308 getKillRegState(KillSrc);
1310 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1311 PPC::GPRCRegClass.contains(DestReg)) {
1312 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1313 getKillRegState(KillSrc);
1315 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1316 PPC::SPERCRegClass.contains(DestReg)) {
1317 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1318 getKillRegState(KillSrc);
1323 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1325 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1327 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1329 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1331 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1333 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1334 // There are two different ways this can be done:
1335 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1336 // issue in VSU pipeline 0.
1337 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1338 // can go to either pipeline.
1339 // We'll always use xxlor here, because in practically all cases where
1340 // copies are generated, they are close enough to some use that the
1341 // lower-latency form is preferable.
1343 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1344 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1345 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1346 else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
1348 else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
1350 else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
1352 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1354 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1357 llvm_unreachable("Impossible reg-to-reg copy");
1359 const MCInstrDesc &MCID = get(Opc);
1360 if (MCID.getNumOperands() == 3)
1361 BuildMI(MBB, I, DL, MCID, DestReg)
1362 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1364 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1367 static unsigned getSpillIndex(const TargetRegisterClass *RC) {
1368 int OpcodeIndex = 0;
1370 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1371 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1372 OpcodeIndex = SOK_Int4Spill;
1373 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1374 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1375 OpcodeIndex = SOK_Int8Spill;
1376 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1377 OpcodeIndex = SOK_Float8Spill;
1378 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1379 OpcodeIndex = SOK_Float4Spill;
1380 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1381 OpcodeIndex = SOK_SPESpill;
1382 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1383 OpcodeIndex = SOK_CRSpill;
1384 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1385 OpcodeIndex = SOK_CRBitSpill;
1386 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1387 OpcodeIndex = SOK_VRVectorSpill;
1388 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1389 OpcodeIndex = SOK_VSXVectorSpill;
1390 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1391 OpcodeIndex = SOK_VectorFloat8Spill;
1392 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1393 OpcodeIndex = SOK_VectorFloat4Spill;
1394 } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
1395 OpcodeIndex = SOK_VRSaveSpill;
1396 } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
1397 OpcodeIndex = SOK_QuadFloat8Spill;
1398 } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
1399 OpcodeIndex = SOK_QuadFloat4Spill;
1400 } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
1401 OpcodeIndex = SOK_QuadBitSpill;
1402 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1403 OpcodeIndex = SOK_SpillToVSR;
1405 llvm_unreachable("Unknown regclass!");
1411 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1412 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1413 return OpcodesForSpill[getSpillIndex(RC)];
1417 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1418 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1419 return OpcodesForSpill[getSpillIndex(RC)];
1422 void PPCInstrInfo::StoreRegToStackSlot(
1423 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1424 const TargetRegisterClass *RC,
1425 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1426 unsigned Opcode = getStoreOpcodeForSpill(RC);
1429 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1430 FuncInfo->setHasSpills();
1432 NewMIs.push_back(addFrameReference(
1433 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1436 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1437 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1438 FuncInfo->setSpillsCR();
1440 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1441 FuncInfo->setSpillsVRSAVE();
1443 if (isXFormMemOp(Opcode))
1444 FuncInfo->setHasNonRISpills();
1447 void PPCInstrInfo::storeRegToStackSlotNoUpd(
1448 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
1449 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1450 const TargetRegisterInfo *TRI) const {
1451 MachineFunction &MF = *MBB.getParent();
1452 SmallVector<MachineInstr *, 4> NewMIs;
1454 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1456 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1457 MBB.insert(MI, NewMIs[i]);
1459 const MachineFrameInfo &MFI = MF.getFrameInfo();
1460 MachineMemOperand *MMO = MF.getMachineMemOperand(
1461 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1462 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1463 MFI.getObjectAlign(FrameIdx));
1464 NewMIs.back()->addMemOperand(MF, MMO);
1467 void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1468 MachineBasicBlock::iterator MI,
1469 Register SrcReg, bool isKill,
1471 const TargetRegisterClass *RC,
1472 const TargetRegisterInfo *TRI) const {
1473 // We need to avoid a situation in which the value from a VRRC register is
1474 // spilled using an Altivec instruction and reloaded into a VSRC register
1475 // using a VSX instruction. The issue with this is that the VSX
1476 // load/store instructions swap the doublewords in the vector and the Altivec
1477 // ones don't. The register classes on the spill/reload may be different if
1478 // the register is defined using an Altivec instruction and is then used by a
1481 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1484 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1485 unsigned DestReg, int FrameIdx,
1486 const TargetRegisterClass *RC,
1487 SmallVectorImpl<MachineInstr *> &NewMIs)
1489 unsigned Opcode = getLoadOpcodeForSpill(RC);
1490 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1492 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1494 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1495 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1496 FuncInfo->setSpillsCR();
1498 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1499 FuncInfo->setSpillsVRSAVE();
1501 if (isXFormMemOp(Opcode))
1502 FuncInfo->setHasNonRISpills();
1505 void PPCInstrInfo::loadRegFromStackSlotNoUpd(
1506 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
1507 int FrameIdx, const TargetRegisterClass *RC,
1508 const TargetRegisterInfo *TRI) const {
1509 MachineFunction &MF = *MBB.getParent();
1510 SmallVector<MachineInstr*, 4> NewMIs;
1512 if (MI != MBB.end()) DL = MI->getDebugLoc();
1514 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1515 FuncInfo->setHasSpills();
1517 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
1519 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1520 MBB.insert(MI, NewMIs[i]);
1522 const MachineFrameInfo &MFI = MF.getFrameInfo();
1523 MachineMemOperand *MMO = MF.getMachineMemOperand(
1524 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1525 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
1526 MFI.getObjectAlign(FrameIdx));
1527 NewMIs.back()->addMemOperand(MF, MMO);
1530 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1531 MachineBasicBlock::iterator MI,
1532 Register DestReg, int FrameIdx,
1533 const TargetRegisterClass *RC,
1534 const TargetRegisterInfo *TRI) const {
1535 // We need to avoid a situation in which the value from a VRRC register is
1536 // spilled using an Altivec instruction and reloaded into a VSRC register
1537 // using a VSX instruction. The issue with this is that the VSX
1538 // load/store instructions swap the doublewords in the vector and the Altivec
1539 // ones don't. The register classes on the spill/reload may be different if
1540 // the register is defined using an Altivec instruction and is then used by a
1544 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
1548 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
1549 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
1550 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
1551 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
1553 // Leave the CR# the same, but invert the condition.
1554 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
1558 // For some instructions, it is legal to fold ZERO into the RA register field.
1559 // This function performs that fold by replacing the operand with PPC::ZERO,
1560 // it does not consider whether the load immediate zero is no longer in use.
1561 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1562 Register Reg) const {
1563 // A zero immediate should always be loaded with a single li.
1564 unsigned DefOpc = DefMI.getOpcode();
1565 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
1567 if (!DefMI.getOperand(1).isImm())
1569 if (DefMI.getOperand(1).getImm() != 0)
1572 // Note that we cannot here invert the arguments of an isel in order to fold
1573 // a ZERO into what is presented as the second argument. All we have here
1574 // is the condition bit, and that might come from a CR-logical bit operation.
1576 const MCInstrDesc &UseMCID = UseMI.getDesc();
1578 // Only fold into real machine instructions.
1579 if (UseMCID.isPseudo())
1582 // We need to find which of the User's operands is to be folded, that will be
1583 // the operand that matches the given register ID.
1585 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
1586 if (UseMI.getOperand(UseIdx).isReg() &&
1587 UseMI.getOperand(UseIdx).getReg() == Reg)
1590 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
1591 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
1593 const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
1595 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
1596 // register (which might also be specified as a pointer class kind).
1597 if (UseInfo->isLookupPtrRegClass()) {
1598 if (UseInfo->RegClass /* Kind */ != 1)
1601 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
1602 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
1606 // Make sure this is not tied to an output register (or otherwise
1607 // constrained). This is true for ST?UX registers, for example, which
1608 // are tied to their output registers.
1609 if (UseInfo->Constraints != 0)
1613 if (UseInfo->isLookupPtrRegClass()) {
1614 bool isPPC64 = Subtarget.isPPC64();
1615 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
1617 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
1618 PPC::ZERO8 : PPC::ZERO;
1621 UseMI.getOperand(UseIdx).setReg(ZeroReg);
1625 // Folds zero into instructions which have a load immediate zero as an operand
1626 // but also recognize zero as immediate zero. If the definition of the load
1627 // has no more users it is deleted.
1628 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1629 Register Reg, MachineRegisterInfo *MRI) const {
1630 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
1631 if (MRI->use_nodbg_empty(Reg))
1632 DefMI.eraseFromParent();
1636 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
1637 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
1639 if (I->definesRegister(PPC::CTR) || I->definesRegister(PPC::CTR8))
1644 // We should make sure that, if we're going to predicate both sides of a
1645 // condition (a diamond), that both sides don't define the counter register. We
1646 // can predicate counter-decrement-based branches, but while that predicates
1647 // the branching, it does not predicate the counter decrement. If we tried to
1648 // merge the triangle into one predicated block, we'd decrement the counter
1650 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
1651 unsigned NumT, unsigned ExtraT,
1652 MachineBasicBlock &FMBB,
1653 unsigned NumF, unsigned ExtraF,
1654 BranchProbability Probability) const {
1655 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
1659 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
1660 // The predicated branches are identified by their type, not really by the
1661 // explicit presence of a predicate. Furthermore, some of them can be
1662 // predicated more than once. Because if conversion won't try to predicate
1663 // any instruction which already claims to be predicated (by returning true
1664 // here), always return false. In doing so, we let isPredicable() be the
1665 // final word on whether not the instruction can be (further) predicated.
1670 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
1671 ArrayRef<MachineOperand> Pred) const {
1672 unsigned OpC = MI.getOpcode();
1673 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
1674 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1675 bool isPPC64 = Subtarget.isPPC64();
1676 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
1677 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
1678 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1679 MI.setDesc(get(PPC::BCLR));
1680 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1681 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1682 MI.setDesc(get(PPC::BCLRn));
1683 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1685 MI.setDesc(get(PPC::BCCLR));
1686 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1687 .addImm(Pred[0].getImm())
1692 } else if (OpC == PPC::B) {
1693 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1694 bool isPPC64 = Subtarget.isPPC64();
1695 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
1696 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
1697 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1698 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1699 MI.RemoveOperand(0);
1701 MI.setDesc(get(PPC::BC));
1702 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1705 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1706 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1707 MI.RemoveOperand(0);
1709 MI.setDesc(get(PPC::BCn));
1710 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1714 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1715 MI.RemoveOperand(0);
1717 MI.setDesc(get(PPC::BCC));
1718 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1719 .addImm(Pred[0].getImm())
1725 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
1726 OpC == PPC::BCTRL8) {
1727 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
1728 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
1730 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8;
1731 bool isPPC64 = Subtarget.isPPC64();
1733 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1734 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
1735 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
1736 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1738 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1739 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
1740 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
1741 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1745 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
1746 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
1747 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1748 .addImm(Pred[0].getImm())
1756 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
1757 ArrayRef<MachineOperand> Pred2) const {
1758 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
1759 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
1761 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
1763 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
1766 // P1 can only subsume P2 if they test the same condition register.
1767 if (Pred1[1].getReg() != Pred2[1].getReg())
1770 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
1771 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
1776 // Does P1 subsume P2, e.g. GE subsumes GT.
1777 if (P1 == PPC::PRED_LE &&
1778 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
1780 if (P1 == PPC::PRED_GE &&
1781 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
1787 bool PPCInstrInfo::DefinesPredicate(MachineInstr &MI,
1788 std::vector<MachineOperand> &Pred) const {
1789 // Note: At the present time, the contents of Pred from this function is
1790 // unused by IfConversion. This implementation follows ARM by pushing the
1791 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
1792 // predicate, instructions defining CTR or CTR8 are also included as
1793 // predicate-defining instructions.
1795 const TargetRegisterClass *RCs[] =
1796 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
1797 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
1800 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1801 const MachineOperand &MO = MI.getOperand(i);
1802 for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) {
1803 const TargetRegisterClass *RC = RCs[c];
1805 if (MO.isDef() && RC->contains(MO.getReg())) {
1809 } else if (MO.isRegMask()) {
1810 for (TargetRegisterClass::iterator I = RC->begin(),
1811 IE = RC->end(); I != IE; ++I)
1812 if (MO.clobbersPhysReg(*I)) {
1823 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1824 Register &SrcReg2, int &Mask,
1826 unsigned Opc = MI.getOpcode();
1829 default: return false;
1834 SrcReg = MI.getOperand(1).getReg();
1836 Value = MI.getOperand(2).getImm();
1845 SrcReg = MI.getOperand(1).getReg();
1846 SrcReg2 = MI.getOperand(2).getReg();
1853 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
1854 Register SrcReg2, int Mask, int Value,
1855 const MachineRegisterInfo *MRI) const {
1859 int OpC = CmpInstr.getOpcode();
1860 Register CRReg = CmpInstr.getOperand(0).getReg();
1862 // FP record forms set CR1 based on the exception status bits, not a
1863 // comparison with zero.
1864 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
1867 const TargetRegisterInfo *TRI = &getRegisterInfo();
1868 // The record forms set the condition register based on a signed comparison
1869 // with zero (so says the ISA manual). This is not as straightforward as it
1870 // seems, however, because this is always a 64-bit comparison on PPC64, even
1871 // for instructions that are 32-bit in nature (like slw for example).
1872 // So, on PPC32, for unsigned comparisons, we can use the record forms only
1873 // for equality checks (as those don't depend on the sign). On PPC64,
1874 // we are restricted to equality for unsigned 64-bit comparisons and for
1875 // signed 32-bit comparisons the applicability is more restricted.
1876 bool isPPC64 = Subtarget.isPPC64();
1877 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
1878 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
1879 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
1881 // Look through copies unless that gets us to a physical register.
1882 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
1883 if (ActualSrc.isVirtual())
1886 // Get the unique definition of SrcReg.
1887 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1888 if (!MI) return false;
1890 bool equalityOnly = false;
1893 if (is32BitSignedCompare) {
1894 // We can perform this optimization only if MI is sign-extending.
1895 if (isSignExtended(*MI))
1899 } else if (is32BitUnsignedCompare) {
1900 // We can perform this optimization, equality only, if MI is
1902 if (isZeroExtended(*MI)) {
1904 equalityOnly = true;
1908 equalityOnly = is64BitUnsignedCompare;
1910 equalityOnly = is32BitUnsignedCompare;
1913 // We need to check the uses of the condition register in order to reject
1914 // non-equality comparisons.
1915 for (MachineRegisterInfo::use_instr_iterator
1916 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
1918 MachineInstr *UseMI = &*I;
1919 if (UseMI->getOpcode() == PPC::BCC) {
1920 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1921 unsigned PredCond = PPC::getPredicateCondition(Pred);
1922 // We ignore hint bits when checking for non-equality comparisons.
1923 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
1925 } else if (UseMI->getOpcode() == PPC::ISEL ||
1926 UseMI->getOpcode() == PPC::ISEL8) {
1927 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
1928 if (SubIdx != PPC::sub_eq)
1935 MachineBasicBlock::iterator I = CmpInstr;
1937 // Scan forward to find the first use of the compare.
1938 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
1940 bool FoundUse = false;
1941 for (MachineRegisterInfo::use_instr_iterator
1942 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
1953 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
1954 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
1956 // There are two possible candidates which can be changed to set CR[01].
1957 // One is MI, the other is a SUB instruction.
1958 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
1959 MachineInstr *Sub = nullptr;
1961 // MI is not a candidate for CMPrr.
1963 // FIXME: Conservatively refuse to convert an instruction which isn't in the
1964 // same BB as the comparison. This is to allow the check below to avoid calls
1965 // (and other explicit clobbers); instead we should really check for these
1966 // more explicitly (in at least a few predecessors).
1967 else if (MI->getParent() != CmpInstr.getParent())
1969 else if (Value != 0) {
1970 // The record-form instructions set CR bit based on signed comparison
1971 // against 0. We try to convert a compare against 1 or -1 into a compare
1972 // against 0 to exploit record-form instructions. For example, we change
1973 // the condition "greater than -1" into "greater than or equal to 0"
1974 // and "less than 1" into "less than or equal to 0".
1976 // Since we optimize comparison based on a specific branch condition,
1977 // we don't optimize if condition code is used by more than once.
1978 if (equalityOnly || !MRI->hasOneUse(CRReg))
1981 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
1982 if (UseMI->getOpcode() != PPC::BCC)
1985 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1986 unsigned PredCond = PPC::getPredicateCondition(Pred);
1987 unsigned PredHint = PPC::getPredicateHint(Pred);
1988 int16_t Immed = (int16_t)Value;
1990 // When modifying the condition in the predicate, we propagate hint bits
1991 // from the original predicate to the new one.
1992 if (Immed == -1 && PredCond == PPC::PRED_GT)
1993 // We convert "greater than -1" into "greater than or equal to 0",
1994 // since we are assuming signed comparison by !equalityOnly
1995 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
1996 else if (Immed == -1 && PredCond == PPC::PRED_LE)
1997 // We convert "less than or equal to -1" into "less than 0".
1998 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
1999 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2000 // We convert "less than 1" into "less than or equal to 0".
2001 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2002 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2003 // We convert "greater than or equal to 1" into "greater than 0".
2004 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2008 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred));
2014 // Get ready to iterate backward from CmpInstr.
2015 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2017 for (; I != E && !noSub; --I) {
2018 const MachineInstr &Instr = *I;
2019 unsigned IOpC = Instr.getOpcode();
2021 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2022 Instr.readsRegister(PPC::CR0, TRI)))
2023 // This instruction modifies or uses the record condition register after
2024 // the one we want to change. While we could do this transformation, it
2025 // would likely not be profitable. This transformation removes one
2026 // instruction, and so even forcing RA to generate one move probably
2027 // makes it unprofitable.
2030 // Check whether CmpInstr can be made redundant by the current instruction.
2031 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2032 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2033 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2034 ((Instr.getOperand(1).getReg() == SrcReg &&
2035 Instr.getOperand(2).getReg() == SrcReg2) ||
2036 (Instr.getOperand(1).getReg() == SrcReg2 &&
2037 Instr.getOperand(2).getReg() == SrcReg))) {
2043 // The 'and' is below the comparison instruction.
2047 // Return false if no candidates exist.
2051 // The single candidate is called MI.
2055 int MIOpC = MI->getOpcode();
2056 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2057 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2060 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2061 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2065 // FIXME: On the non-embedded POWER architectures, only some of the record
2066 // forms are fast, and we should use only the fast ones.
2068 // The defining instruction has a record form (or is already a record
2069 // form). It is possible, however, that we'll need to reverse the condition
2070 // code of the users.
2074 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2075 // needs to be updated to be based on SUB. Push the condition code
2076 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2077 // condition code of these operands will be modified.
2078 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2079 // comparison against 0, which may modify predicate.
2080 bool ShouldSwap = false;
2081 if (Sub && Value == 0) {
2082 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2083 Sub->getOperand(2).getReg() == SrcReg;
2085 // The operands to subf are the opposite of sub, so only in the fixed-point
2086 // case, invert the order.
2087 ShouldSwap = !ShouldSwap;
2091 for (MachineRegisterInfo::use_instr_iterator
2092 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2094 MachineInstr *UseMI = &*I;
2095 if (UseMI->getOpcode() == PPC::BCC) {
2096 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
2097 unsigned PredCond = PPC::getPredicateCondition(Pred);
2098 assert((!equalityOnly ||
2099 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2100 "Invalid predicate for equality-only optimization");
2101 (void)PredCond; // To suppress warning in release build.
2102 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2103 PPC::getSwappedPredicate(Pred)));
2104 } else if (UseMI->getOpcode() == PPC::ISEL ||
2105 UseMI->getOpcode() == PPC::ISEL8) {
2106 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2107 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2108 "Invalid CR bit for equality-only optimization");
2110 if (NewSubReg == PPC::sub_lt)
2111 NewSubReg = PPC::sub_gt;
2112 else if (NewSubReg == PPC::sub_gt)
2113 NewSubReg = PPC::sub_lt;
2115 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2117 } else // We need to abort on a user we don't understand.
2120 assert(!(Value != 0 && ShouldSwap) &&
2121 "Non-zero immediate support and ShouldSwap"
2122 "may conflict in updating predicate");
2124 // Create a new virtual register to hold the value of the CR set by the
2125 // record-form instruction. If the instruction was not previously in
2126 // record form, then set the kill flag on the CR.
2127 CmpInstr.eraseFromParent();
2129 MachineBasicBlock::iterator MII = MI;
2130 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2131 get(TargetOpcode::COPY), CRReg)
2132 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2134 // Even if CR0 register were dead before, it is alive now since the
2135 // instruction we just built uses it.
2136 MI->clearRegisterDeads(PPC::CR0);
2138 if (MIOpC != NewOpC) {
2139 // We need to be careful here: we're replacing one instruction with
2140 // another, and we need to make sure that we get all of the right
2141 // implicit uses and defs. On the other hand, the caller may be holding
2142 // an iterator to this instruction, and so we can't delete it (this is
2143 // specifically the case if this is the instruction directly after the
2146 // Rotates are expensive instructions. If we're emitting a record-form
2147 // rotate that can just be an andi/andis, we should just emit that.
2148 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2149 Register GPRRes = MI->getOperand(0).getReg();
2150 int64_t SH = MI->getOperand(2).getImm();
2151 int64_t MB = MI->getOperand(3).getImm();
2152 int64_t ME = MI->getOperand(4).getImm();
2153 // We can only do this if both the start and end of the mask are in the
2155 bool MBInLoHWord = MB >= 16;
2156 bool MEInLoHWord = ME >= 16;
2157 uint64_t Mask = ~0LLU;
2159 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2160 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2161 // The mask value needs to shift right 16 if we're emitting andis.
2162 Mask >>= MBInLoHWord ? 0 : 16;
2163 NewOpC = MIOpC == PPC::RLWINM
2164 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2165 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2166 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2167 (ME - MB + 1 == SH) && (MB >= 16)) {
2168 // If we are rotating by the exact number of bits as are in the mask
2169 // and the mask is in the least significant bits of the register,
2170 // that's just an andis. (as long as the GPR result has no uses).
2171 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2173 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2175 // If we've set the mask, we can transform.
2176 if (Mask != ~0LLU) {
2177 MI->RemoveOperand(4);
2178 MI->RemoveOperand(3);
2179 MI->getOperand(2).setImm(Mask);
2180 NumRcRotatesConvertedToRcAnd++;
2182 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2183 int64_t MB = MI->getOperand(3).getImm();
2185 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2186 NewOpC = PPC::ANDI8_rec;
2187 MI->RemoveOperand(3);
2188 MI->getOperand(2).setImm(Mask);
2189 NumRcRotatesConvertedToRcAnd++;
2193 const MCInstrDesc &NewDesc = get(NewOpC);
2194 MI->setDesc(NewDesc);
2196 if (NewDesc.ImplicitDefs)
2197 for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
2198 *ImpDefs; ++ImpDefs)
2199 if (!MI->definesRegister(*ImpDefs))
2200 MI->addOperand(*MI->getParent()->getParent(),
2201 MachineOperand::CreateReg(*ImpDefs, true, true));
2202 if (NewDesc.ImplicitUses)
2203 for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
2204 *ImpUses; ++ImpUses)
2205 if (!MI->readsRegister(*ImpUses))
2206 MI->addOperand(*MI->getParent()->getParent(),
2207 MachineOperand::CreateReg(*ImpUses, false, true));
2209 assert(MI->definesRegister(PPC::CR0) &&
2210 "Record-form instruction does not define cr0?");
2212 // Modify the condition code of operands in OperandsToUpdate.
2213 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2214 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2215 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2216 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2218 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2219 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2224 /// GetInstSize - Return the number of bytes of code the specified
2225 /// instruction may be. This returns the maximum number of bytes.
2227 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2228 unsigned Opcode = MI.getOpcode();
2230 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2231 const MachineFunction *MF = MI.getParent()->getParent();
2232 const char *AsmStr = MI.getOperand(0).getSymbolName();
2233 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2234 } else if (Opcode == TargetOpcode::STACKMAP) {
2235 StackMapOpers Opers(&MI);
2236 return Opers.getNumPatchBytes();
2237 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2238 PatchPointOpers Opers(&MI);
2239 return Opers.getNumPatchBytes();
2241 return get(Opcode).getSize();
2245 std::pair<unsigned, unsigned>
2246 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2247 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2248 return std::make_pair(TF & Mask, TF & ~Mask);
2251 ArrayRef<std::pair<unsigned, const char *>>
2252 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2253 using namespace PPCII;
2254 static const std::pair<unsigned, const char *> TargetFlags[] = {
2257 {MO_TPREL_LO, "ppc-tprel-lo"},
2258 {MO_TPREL_HA, "ppc-tprel-ha"},
2259 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2260 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2261 {MO_TOC_LO, "ppc-toc-lo"},
2262 {MO_TLS, "ppc-tls"}};
2263 return makeArrayRef(TargetFlags);
2266 ArrayRef<std::pair<unsigned, const char *>>
2267 PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
2268 using namespace PPCII;
2269 static const std::pair<unsigned, const char *> TargetFlags[] = {
2270 {MO_PLT, "ppc-plt"},
2271 {MO_PIC_FLAG, "ppc-pic"},
2272 {MO_PCREL_FLAG, "ppc-pcrel"},
2273 {MO_GOT_FLAG, "ppc-got"}};
2274 return makeArrayRef(TargetFlags);
2277 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2278 // The VSX versions have the advantage of a full 64-register target whereas
2279 // the FP ones have the advantage of lower latency and higher throughput. So
2280 // what we are after is using the faster instructions in low register pressure
2281 // situations and using the larger register file in high register pressure
2283 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2284 unsigned UpperOpcode, LowerOpcode;
2285 switch (MI.getOpcode()) {
2286 case PPC::DFLOADf32:
2287 UpperOpcode = PPC::LXSSP;
2288 LowerOpcode = PPC::LFS;
2290 case PPC::DFLOADf64:
2291 UpperOpcode = PPC::LXSD;
2292 LowerOpcode = PPC::LFD;
2294 case PPC::DFSTOREf32:
2295 UpperOpcode = PPC::STXSSP;
2296 LowerOpcode = PPC::STFS;
2298 case PPC::DFSTOREf64:
2299 UpperOpcode = PPC::STXSD;
2300 LowerOpcode = PPC::STFD;
2302 case PPC::XFLOADf32:
2303 UpperOpcode = PPC::LXSSPX;
2304 LowerOpcode = PPC::LFSX;
2306 case PPC::XFLOADf64:
2307 UpperOpcode = PPC::LXSDX;
2308 LowerOpcode = PPC::LFDX;
2310 case PPC::XFSTOREf32:
2311 UpperOpcode = PPC::STXSSPX;
2312 LowerOpcode = PPC::STFSX;
2314 case PPC::XFSTOREf64:
2315 UpperOpcode = PPC::STXSDX;
2316 LowerOpcode = PPC::STFDX;
2319 UpperOpcode = PPC::LXSIWAX;
2320 LowerOpcode = PPC::LFIWAX;
2323 UpperOpcode = PPC::LXSIWZX;
2324 LowerOpcode = PPC::LFIWZX;
2327 UpperOpcode = PPC::STXSIWX;
2328 LowerOpcode = PPC::STFIWX;
2331 llvm_unreachable("Unknown Operation!");
2334 Register TargetReg = MI.getOperand(0).getReg();
2336 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
2337 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
2338 Opcode = LowerOpcode;
2340 Opcode = UpperOpcode;
2341 MI.setDesc(get(Opcode));
2345 static bool isAnImmediateOperand(const MachineOperand &MO) {
2346 return MO.isCPI() || MO.isGlobal() || MO.isImm();
2349 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2350 auto &MBB = *MI.getParent();
2351 auto DL = MI.getDebugLoc();
2353 switch (MI.getOpcode()) {
2354 case TargetOpcode::LOAD_STACK_GUARD: {
2355 assert(Subtarget.isTargetLinux() &&
2356 "Only Linux target is expected to contain LOAD_STACK_GUARD");
2357 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
2358 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
2359 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
2360 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2365 case PPC::DFLOADf32:
2366 case PPC::DFLOADf64:
2367 case PPC::DFSTOREf32:
2368 case PPC::DFSTOREf64: {
2369 assert(Subtarget.hasP9Vector() &&
2370 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
2371 assert(MI.getOperand(2).isReg() &&
2372 isAnImmediateOperand(MI.getOperand(1)) &&
2373 "D-form op must have register and immediate operands");
2374 return expandVSXMemPseudo(MI);
2376 case PPC::XFLOADf32:
2377 case PPC::XFSTOREf32:
2381 assert(Subtarget.hasP8Vector() &&
2382 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
2383 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2384 "X-form op must have register and register operands");
2385 return expandVSXMemPseudo(MI);
2387 case PPC::XFLOADf64:
2388 case PPC::XFSTOREf64: {
2389 assert(Subtarget.hasVSX() &&
2390 "Invalid X-Form Pseudo-ops on target that has no VSX.");
2391 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2392 "X-form op must have register and register operands");
2393 return expandVSXMemPseudo(MI);
2395 case PPC::SPILLTOVSR_LD: {
2396 Register TargetReg = MI.getOperand(0).getReg();
2397 if (PPC::VSFRCRegClass.contains(TargetReg)) {
2398 MI.setDesc(get(PPC::DFLOADf64));
2399 return expandPostRAPseudo(MI);
2402 MI.setDesc(get(PPC::LD));
2405 case PPC::SPILLTOVSR_ST: {
2406 Register SrcReg = MI.getOperand(0).getReg();
2407 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2408 NumStoreSPILLVSRRCAsVec++;
2409 MI.setDesc(get(PPC::DFSTOREf64));
2410 return expandPostRAPseudo(MI);
2412 NumStoreSPILLVSRRCAsGpr++;
2413 MI.setDesc(get(PPC::STD));
2417 case PPC::SPILLTOVSR_LDX: {
2418 Register TargetReg = MI.getOperand(0).getReg();
2419 if (PPC::VSFRCRegClass.contains(TargetReg))
2420 MI.setDesc(get(PPC::LXSDX));
2422 MI.setDesc(get(PPC::LDX));
2425 case PPC::SPILLTOVSR_STX: {
2426 Register SrcReg = MI.getOperand(0).getReg();
2427 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2428 NumStoreSPILLVSRRCAsVec++;
2429 MI.setDesc(get(PPC::STXSDX));
2431 NumStoreSPILLVSRRCAsGpr++;
2432 MI.setDesc(get(PPC::STDX));
2437 case PPC::CFENCE8: {
2438 auto Val = MI.getOperand(0).getReg();
2439 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
2440 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
2441 .addImm(PPC::PRED_NE_MINUS)
2444 MI.setDesc(get(PPC::ISYNC));
2445 MI.RemoveOperand(0);
2452 // Essentially a compile-time implementation of a compare->isel sequence.
2453 // It takes two constants to compare, along with the true/false registers
2454 // and the comparison type (as a subreg to a CR field) and returns one
2455 // of the true/false registers, depending on the comparison results.
2456 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
2457 unsigned TrueReg, unsigned FalseReg,
2458 unsigned CRSubReg) {
2459 // Signed comparisons. The immediates are assumed to be sign-extended.
2460 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
2462 default: llvm_unreachable("Unknown integer comparison type.");
2464 return Imm1 < Imm2 ? TrueReg : FalseReg;
2466 return Imm1 > Imm2 ? TrueReg : FalseReg;
2468 return Imm1 == Imm2 ? TrueReg : FalseReg;
2471 // Unsigned comparisons.
2472 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
2474 default: llvm_unreachable("Unknown integer comparison type.");
2476 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
2478 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
2480 return Imm1 == Imm2 ? TrueReg : FalseReg;
2483 return PPC::NoRegister;
2486 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
2488 int64_t Imm) const {
2489 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
2490 // Replace the REG with the Immediate.
2491 Register InUseReg = MI.getOperand(OpNo).getReg();
2492 MI.getOperand(OpNo).ChangeToImmediate(Imm);
2494 if (MI.implicit_operands().empty())
2497 // We need to make sure that the MI didn't have any implicit use
2498 // of this REG any more.
2499 const TargetRegisterInfo *TRI = &getRegisterInfo();
2500 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
2501 if (UseOpIdx >= 0) {
2502 MachineOperand &MO = MI.getOperand(UseOpIdx);
2503 if (MO.isImplicit())
2504 // The operands must always be in the following order:
2505 // - explicit reg defs,
2506 // - other explicit operands (reg uses, immediates, etc.),
2507 // - implicit reg defs
2508 // - implicit reg uses
2509 // Therefore, removing the implicit operand won't change the explicit
2511 MI.RemoveOperand(UseOpIdx);
2515 // Replace an instruction with one that materializes a constant (and sets
2516 // CR0 if the original instruction was a record-form instruction).
2517 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
2518 const LoadImmediateInfo &LII) const {
2519 // Remove existing operands.
2520 int OperandToKeep = LII.SetCR ? 1 : 0;
2521 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
2522 MI.RemoveOperand(i);
2524 // Replace the instruction.
2526 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
2527 // Set the immediate.
2528 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2529 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
2533 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
2535 // Set the immediate.
2536 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2540 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
2541 bool &SeenIntermediateUse) const {
2542 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
2543 "Should be called after register allocation.");
2544 const TargetRegisterInfo *TRI = &getRegisterInfo();
2545 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
2547 SeenIntermediateUse = false;
2548 for (; It != E; ++It) {
2549 if (It->modifiesRegister(Reg, TRI))
2551 if (It->readsRegister(Reg, TRI))
2552 SeenIntermediateUse = true;
2557 MachineInstr *PPCInstrInfo::getForwardingDefMI(
2559 unsigned &OpNoForForwarding,
2560 bool &SeenIntermediateUse) const {
2561 OpNoForForwarding = ~0U;
2562 MachineInstr *DefMI = nullptr;
2563 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
2564 const TargetRegisterInfo *TRI = &getRegisterInfo();
2565 // If we're in SSA, get the defs through the MRI. Otherwise, only look
2566 // within the basic block to see if the register is defined using an
2567 // LI/LI8/ADDI/ADDI8.
2569 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2570 if (!MI.getOperand(i).isReg())
2572 Register Reg = MI.getOperand(i).getReg();
2573 if (!Register::isVirtualRegister(Reg))
2575 unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI);
2576 if (Register::isVirtualRegister(TrueReg)) {
2577 DefMI = MRI->getVRegDef(TrueReg);
2578 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
2579 DefMI->getOpcode() == PPC::ADDI ||
2580 DefMI->getOpcode() == PPC::ADDI8) {
2581 OpNoForForwarding = i;
2582 // The ADDI and LI operand maybe exist in one instruction at same
2583 // time. we prefer to fold LI operand as LI only has one Imm operand
2584 // and is more possible to be converted. So if current DefMI is
2585 // ADDI/ADDI8, we continue to find possible LI/LI8.
2586 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
2592 // Looking back through the definition for each operand could be expensive,
2593 // so exit early if this isn't an instruction that either has an immediate
2594 // form or is already an immediate form that we can handle.
2596 unsigned Opc = MI.getOpcode();
2597 bool ConvertibleImmForm =
2598 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
2599 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
2600 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
2601 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
2602 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
2603 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
2604 Opc == PPC::RLWINM8_rec;
2605 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
2606 ? isVFRegister(MI.getOperand(0).getReg())
2608 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
2611 // Don't convert or %X, %Y, %Y since that's just a register move.
2612 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
2613 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
2615 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2616 MachineOperand &MO = MI.getOperand(i);
2617 SeenIntermediateUse = false;
2618 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
2619 Register Reg = MI.getOperand(i).getReg();
2620 // If we see another use of this reg between the def and the MI,
2621 // we want to flat it so the def isn't deleted.
2622 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
2624 // Is this register defined by some form of add-immediate (including
2625 // load-immediate) within this basic block?
2626 switch (DefMI->getOpcode()) {
2634 OpNoForForwarding = i;
2641 return OpNoForForwarding == ~0U ? nullptr : DefMI;
2644 unsigned PPCInstrInfo::getSpillTarget() const {
2645 return Subtarget.hasP9Vector() ? 1 : 0;
2648 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const {
2649 return StoreSpillOpcodesArray[getSpillTarget()];
2652 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
2653 return LoadSpillOpcodesArray[getSpillTarget()];
2656 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr *StartMI, MachineInstr *EndMI,
2657 unsigned RegNo) const {
2658 // Conservatively clear kill flag for the register if the instructions are in
2659 // different basic blocks and in SSA form, because the kill flag may no longer
2660 // be right. There is no need to bother with dead flags since defs with no
2661 // uses will be handled by DCE.
2662 MachineRegisterInfo &MRI = StartMI->getParent()->getParent()->getRegInfo();
2663 if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) {
2664 MRI.clearKillFlags(RegNo);
2668 // Instructions between [StartMI, EndMI] should be in same basic block.
2669 assert((StartMI->getParent() == EndMI->getParent()) &&
2670 "Instructions are not in same basic block");
2672 // If before RA, StartMI may be def through COPY, we need to adjust it to the
2673 // real def. See function getForwardingDefMI.
2676 std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo);
2677 if (!Reads && !Writes) {
2678 assert(Register::isVirtualRegister(RegNo) &&
2679 "Must be a virtual register");
2680 // Get real def and ignore copies.
2681 StartMI = MRI.getVRegDef(RegNo);
2685 bool IsKillSet = false;
2687 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
2688 MachineOperand &MO = MI.getOperand(Index);
2689 if (MO.isReg() && MO.isUse() && MO.isKill() &&
2690 getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
2691 MO.setIsKill(false);
2694 // Set killed flag for EndMI.
2695 // No need to do anything if EndMI defines RegNo.
2697 EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
2698 if (UseIndex != -1) {
2699 EndMI->getOperand(UseIndex).setIsKill(true);
2701 // Clear killed flag for other EndMI operands related to RegNo. In some
2702 // upexpected cases, killed may be set multiple times for same register
2703 // operand in same MI.
2704 for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i)
2706 clearOperandKillInfo(*EndMI, i);
2709 // Walking the inst in reverse order (EndMI -> StartMI].
2710 MachineBasicBlock::reverse_iterator It = *EndMI;
2711 MachineBasicBlock::reverse_iterator E = EndMI->getParent()->rend();
2712 // EndMI has been handled above, skip it here.
2714 MachineOperand *MO = nullptr;
2715 for (; It != E; ++It) {
2716 // Skip insturctions which could not be a def/use of RegNo.
2717 if (It->isDebugInstr() || It->isPosition())
2720 // Clear killed flag for all It operands related to RegNo. In some
2721 // upexpected cases, killed may be set multiple times for same register
2722 // operand in same MI.
2723 for (int i = 0, e = It->getNumOperands(); i != e; ++i)
2724 clearOperandKillInfo(*It, i);
2726 // If killed is not set, set killed for its last use or set dead for its def
2729 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
2730 // Use found, set it killed.
2732 MO->setIsKill(true);
2734 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
2735 &getRegisterInfo()))) {
2736 // No use found, set dead for its def.
2737 assert(&*It == StartMI && "No new def between StartMI and EndMI.");
2738 MO->setIsDead(true);
2743 if ((&*It) == StartMI)
2746 // Ensure RegMo liveness is killed after EndMI.
2747 assert((IsKillSet || (MO && MO->isDead())) &&
2748 "RegNo should be killed or dead");
2751 // This opt tries to convert the following imm form to an index form to save an
2752 // add for stack variables.
2753 // Return false if no such pattern found.
2755 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2756 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
2757 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
2759 // can be converted to:
2761 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
2762 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
2764 // In order to eliminate ADD instr, make sure that:
2765 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
2766 // new ADDI instr and ADDI can only take int16 Imm.
2767 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
2768 // between ADDI and ADD instr since its original def in ADDI will be changed
2769 // in new ADDI instr. And also there should be no new def for it between
2770 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
2771 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
2772 // between ADD and Imm instr since ADD instr will be eliminated.
2773 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
2774 // moved to Index instr.
2775 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
2776 MachineFunction *MF = MI.getParent()->getParent();
2777 MachineRegisterInfo *MRI = &MF->getRegInfo();
2778 bool PostRA = !MRI->isSSA();
2779 // Do this opt after PEI which is after RA. The reason is stack slot expansion
2780 // in PEI may expose such opportunities since in PEI, stack slot offsets to
2781 // frame base(OffsetAddi) are determined.
2784 unsigned ToBeDeletedReg = 0;
2785 int64_t OffsetImm = 0;
2786 unsigned XFormOpcode = 0;
2789 // Check if Imm instr meets requirement.
2790 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
2794 bool OtherIntermediateUse = false;
2795 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
2797 // Exit if there is other use between ADD and Imm instr or no def found.
2798 if (OtherIntermediateUse || !ADDMI)
2801 // Check if ADD instr meets requirement.
2802 if (!isADDInstrEligibleForFolding(*ADDMI))
2805 unsigned ScaleRegIdx = 0;
2806 int64_t OffsetAddi = 0;
2807 MachineInstr *ADDIMI = nullptr;
2809 // Check if there is a valid ToBeChangedReg in ADDMI.
2810 // 1: It must be killed.
2811 // 2: Its definition must be a valid ADDIMI.
2812 // 3: It must satify int16 offset requirement.
2813 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
2815 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
2820 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
2821 unsigned ToBeChangedReg = ADDIMI->getOperand(0).getReg();
2822 unsigned ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
2823 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
2824 MachineBasicBlock::iterator End) {
2825 for (auto It = ++Start; It != End; It++)
2826 if (It->modifiesRegister(Reg, &getRegisterInfo()))
2831 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
2832 // treated as special zero when ScaleReg is R0/X0 register.
2833 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
2834 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
2837 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
2839 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
2842 // Now start to do the transformation.
2843 LLVM_DEBUG(dbgs() << "Replace instruction: "
2845 LLVM_DEBUG(ADDIMI->dump());
2846 LLVM_DEBUG(ADDMI->dump());
2847 LLVM_DEBUG(MI.dump());
2848 LLVM_DEBUG(dbgs() << "with: "
2851 // Update ADDI instr.
2852 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
2854 // Update Imm instr.
2855 MI.setDesc(get(XFormOpcode));
2856 MI.getOperand(III.ImmOpNo)
2857 .ChangeToRegister(ScaleReg, false, false,
2858 ADDMI->getOperand(ScaleRegIdx).isKill());
2860 MI.getOperand(III.OpNoForForwarding)
2861 .ChangeToRegister(ToBeChangedReg, false, false, true);
2863 // Eliminate ADD instr.
2864 ADDMI->eraseFromParent();
2866 LLVM_DEBUG(ADDIMI->dump());
2867 LLVM_DEBUG(MI.dump());
2872 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
2873 int64_t &Imm) const {
2874 unsigned Opc = ADDIMI.getOpcode();
2876 // Exit if the instruction is not ADDI.
2877 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
2880 // The operand may not necessarily be an immediate - it could be a relocation.
2881 if (!ADDIMI.getOperand(2).isImm())
2884 Imm = ADDIMI.getOperand(2).getImm();
2889 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
2890 unsigned Opc = ADDMI.getOpcode();
2892 // Exit if the instruction is not ADD.
2893 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
2896 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
2897 unsigned &ToBeDeletedReg,
2898 unsigned &XFormOpcode,
2900 ImmInstrInfo &III) const {
2901 // Only handle load/store.
2902 if (!MI.mayLoadOrStore())
2905 unsigned Opc = MI.getOpcode();
2907 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
2909 // Exit if instruction has no index form.
2910 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
2913 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
2914 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
2918 if (!III.IsSummingOperands)
2921 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
2922 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
2923 // Only support imm operands, not relocation slots or others.
2924 if (!ImmOperand.isImm())
2927 assert(RegOperand.isReg() && "Instruction format is not right");
2929 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
2930 if (!RegOperand.isKill())
2933 ToBeDeletedReg = RegOperand.getReg();
2934 OffsetImm = ImmOperand.getImm();
2939 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
2940 MachineInstr *&ADDIMI,
2941 int64_t &OffsetAddi,
2942 int64_t OffsetImm) const {
2943 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
2944 MachineOperand &MO = ADDMI->getOperand(Index);
2949 bool OtherIntermediateUse = false;
2951 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
2952 // Currently handle only one "add + Imminstr" pair case, exit if other
2953 // intermediate use for ToBeChangedReg found.
2954 // TODO: handle the cases where there are other "add + Imminstr" pairs
2955 // with same offset in Imminstr which is like:
2957 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2958 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
2959 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
2960 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
2961 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
2963 // can be converted to:
2965 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
2966 // (OffsetAddi + OffsetImm)
2967 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
2968 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
2970 if (OtherIntermediateUse || !ADDIMI)
2972 // Check if ADDI instr meets requirement.
2973 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
2976 if (isInt<16>(OffsetAddi + OffsetImm))
2981 // If this instruction has an immediate form and one of its operands is a
2982 // result of a load-immediate or an add-immediate, convert it to
2983 // the immediate form if the constant is in range.
2984 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
2985 MachineInstr **KilledDef) const {
2986 MachineFunction *MF = MI.getParent()->getParent();
2987 MachineRegisterInfo *MRI = &MF->getRegInfo();
2988 bool PostRA = !MRI->isSSA();
2989 bool SeenIntermediateUse = true;
2990 unsigned ForwardingOperand = ~0U;
2991 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
2992 SeenIntermediateUse);
2995 assert(ForwardingOperand < MI.getNumOperands() &&
2996 "The forwarding operand needs to be valid at this point");
2997 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
2998 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
2999 if (KilledDef && KillFwdDefMI)
3002 // If this is a imm instruction and its register operands is produced by ADDI,
3003 // put the imm into imm inst directly.
3004 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3005 PPC::INSTRUCTION_LIST_END &&
3006 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3010 bool IsVFReg = MI.getOperand(0).isReg()
3011 ? isVFRegister(MI.getOperand(0).getReg())
3013 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3014 // If this is a reg+reg instruction that has a reg+imm form,
3015 // and one of the operands is produced by an add-immediate,
3016 // try to convert it.
3018 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3022 // If this is a reg+reg instruction that has a reg+imm form,
3023 // and one of the operands is produced by LI, convert it now.
3025 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3028 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3029 // can be simpified to LI.
3030 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3036 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3037 ImmInstrInfo &III, bool PostRA) const {
3038 // The vast majority of the instructions would need their operand 2 replaced
3039 // with an immediate when switching to the reg+imm form. A marked exception
3040 // are the update form loads/stores for which a constant operand 2 would need
3041 // to turn into a displacement and move operand 1 to the operand 2 position.
3043 III.OpNoForForwarding = 2;
3045 III.ImmMustBeMultipleOf = 1;
3046 III.TruncateImmTo = 0;
3047 III.IsSummingOperands = false;
3049 default: return false;
3052 III.SignedImm = true;
3053 III.ZeroIsSpecialOrig = 0;
3054 III.ZeroIsSpecialNew = 1;
3055 III.IsCommutative = true;
3056 III.IsSummingOperands = true;
3057 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3061 III.SignedImm = true;
3062 III.ZeroIsSpecialOrig = 0;
3063 III.ZeroIsSpecialNew = 0;
3064 III.IsCommutative = true;
3065 III.IsSummingOperands = true;
3066 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3069 III.SignedImm = true;
3070 III.ZeroIsSpecialOrig = 0;
3071 III.ZeroIsSpecialNew = 0;
3072 III.IsCommutative = true;
3073 III.IsSummingOperands = true;
3074 III.ImmOpcode = PPC::ADDIC_rec;
3078 III.SignedImm = true;
3079 III.ZeroIsSpecialOrig = 0;
3080 III.ZeroIsSpecialNew = 0;
3081 III.IsCommutative = false;
3082 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3086 III.SignedImm = true;
3087 III.ZeroIsSpecialOrig = 0;
3088 III.ZeroIsSpecialNew = 0;
3089 III.IsCommutative = false;
3090 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3094 III.SignedImm = false;
3095 III.ZeroIsSpecialOrig = 0;
3096 III.ZeroIsSpecialNew = 0;
3097 III.IsCommutative = false;
3098 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3106 III.SignedImm = false;
3107 III.ZeroIsSpecialOrig = 0;
3108 III.ZeroIsSpecialNew = 0;
3109 III.IsCommutative = true;
3111 default: llvm_unreachable("Unknown opcode");
3113 III.ImmOpcode = PPC::ANDI_rec;
3116 III.ImmOpcode = PPC::ANDI8_rec;
3118 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3119 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3120 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3121 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3126 case PPC::RLWNM_rec:
3127 case PPC::RLWNM8_rec:
3138 III.SignedImm = false;
3139 III.ZeroIsSpecialOrig = 0;
3140 III.ZeroIsSpecialNew = 0;
3141 III.IsCommutative = false;
3142 // This isn't actually true, but the instructions ignore any of the
3143 // upper bits, so any immediate loaded with an LI is acceptable.
3144 // This does not apply to shift right algebraic because a value
3145 // out of range will produce a -1/0.
3147 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3148 Opc == PPC::RLWNM8_rec)
3149 III.TruncateImmTo = 5;
3151 III.TruncateImmTo = 6;
3153 default: llvm_unreachable("Unknown opcode");
3154 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3155 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3156 case PPC::RLWNM_rec:
3157 III.ImmOpcode = PPC::RLWINM_rec;
3159 case PPC::RLWNM8_rec:
3160 III.ImmOpcode = PPC::RLWINM8_rec;
3162 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
3163 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
3165 III.ImmOpcode = PPC::RLWINM_rec;
3168 III.ImmOpcode = PPC::RLWINM8_rec;
3170 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
3171 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
3173 III.ImmOpcode = PPC::RLWINM_rec;
3176 III.ImmOpcode = PPC::RLWINM8_rec;
3180 III.TruncateImmTo = 0;
3181 III.ImmOpcode = PPC::SRAWI;
3185 III.TruncateImmTo = 0;
3186 III.ImmOpcode = PPC::SRAWI_rec;
3191 case PPC::RLDCL_rec:
3193 case PPC::RLDCR_rec:
3200 III.SignedImm = false;
3201 III.ZeroIsSpecialOrig = 0;
3202 III.ZeroIsSpecialNew = 0;
3203 III.IsCommutative = false;
3204 // This isn't actually true, but the instructions ignore any of the
3205 // upper bits, so any immediate loaded with an LI is acceptable.
3206 // This does not apply to shift right algebraic because a value
3207 // out of range will produce a -1/0.
3209 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
3210 Opc == PPC::RLDCR_rec)
3211 III.TruncateImmTo = 6;
3213 III.TruncateImmTo = 7;
3215 default: llvm_unreachable("Unknown opcode");
3216 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
3217 case PPC::RLDCL_rec:
3218 III.ImmOpcode = PPC::RLDICL_rec;
3220 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
3221 case PPC::RLDCR_rec:
3222 III.ImmOpcode = PPC::RLDICR_rec;
3224 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
3226 III.ImmOpcode = PPC::RLDICR_rec;
3228 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
3230 III.ImmOpcode = PPC::RLDICL_rec;
3234 III.TruncateImmTo = 0;
3235 III.ImmOpcode = PPC::SRADI;
3239 III.TruncateImmTo = 0;
3240 III.ImmOpcode = PPC::SRADI_rec;
3244 // Loads and stores:
3266 III.SignedImm = true;
3267 III.ZeroIsSpecialOrig = 1;
3268 III.ZeroIsSpecialNew = 2;
3269 III.IsCommutative = true;
3270 III.IsSummingOperands = true;
3272 III.OpNoForForwarding = 2;
3274 default: llvm_unreachable("Unknown opcode");
3275 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
3276 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
3277 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
3278 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
3279 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
3280 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
3281 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
3282 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
3284 III.ImmOpcode = PPC::LWA;
3285 III.ImmMustBeMultipleOf = 4;
3287 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
3288 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
3289 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
3290 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
3291 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
3292 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
3293 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
3294 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
3295 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
3297 III.ImmOpcode = PPC::STD;
3298 III.ImmMustBeMultipleOf = 4;
3300 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
3301 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
3324 III.SignedImm = true;
3325 III.ZeroIsSpecialOrig = 2;
3326 III.ZeroIsSpecialNew = 3;
3327 III.IsCommutative = false;
3328 III.IsSummingOperands = true;
3330 III.OpNoForForwarding = 3;
3332 default: llvm_unreachable("Unknown opcode");
3333 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
3334 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
3335 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
3336 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
3337 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
3338 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
3339 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
3340 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
3342 III.ImmOpcode = PPC::LDU;
3343 III.ImmMustBeMultipleOf = 4;
3345 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
3346 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
3347 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
3348 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
3349 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
3350 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
3351 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
3352 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
3354 III.ImmOpcode = PPC::STDU;
3355 III.ImmMustBeMultipleOf = 4;
3357 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
3358 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
3361 // Power9 and up only. For some of these, the X-Form version has access to all
3362 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
3363 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
3364 // into or stored from is one of the VR registers.
3371 case PPC::XFLOADf32:
3372 case PPC::XFLOADf64:
3373 case PPC::XFSTOREf32:
3374 case PPC::XFSTOREf64:
3375 if (!Subtarget.hasP9Vector())
3377 III.SignedImm = true;
3378 III.ZeroIsSpecialOrig = 1;
3379 III.ZeroIsSpecialNew = 2;
3380 III.IsCommutative = true;
3381 III.IsSummingOperands = true;
3383 III.OpNoForForwarding = 2;
3384 III.ImmMustBeMultipleOf = 4;
3386 default: llvm_unreachable("Unknown opcode");
3388 III.ImmOpcode = PPC::LXV;
3389 III.ImmMustBeMultipleOf = 16;
3394 III.ImmOpcode = PPC::LXSSP;
3396 III.ImmOpcode = PPC::LFS;
3397 III.ImmMustBeMultipleOf = 1;
3402 case PPC::XFLOADf32:
3403 III.ImmOpcode = PPC::DFLOADf32;
3408 III.ImmOpcode = PPC::LXSD;
3410 III.ImmOpcode = PPC::LFD;
3411 III.ImmMustBeMultipleOf = 1;
3416 case PPC::XFLOADf64:
3417 III.ImmOpcode = PPC::DFLOADf64;
3420 III.ImmOpcode = PPC::STXV;
3421 III.ImmMustBeMultipleOf = 16;
3426 III.ImmOpcode = PPC::STXSSP;
3428 III.ImmOpcode = PPC::STFS;
3429 III.ImmMustBeMultipleOf = 1;
3434 case PPC::XFSTOREf32:
3435 III.ImmOpcode = PPC::DFSTOREf32;
3440 III.ImmOpcode = PPC::STXSD;
3442 III.ImmOpcode = PPC::STFD;
3443 III.ImmMustBeMultipleOf = 1;
3448 case PPC::XFSTOREf64:
3449 III.ImmOpcode = PPC::DFSTOREf64;
3457 // Utility function for swaping two arbitrary operands of an instruction.
3458 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
3459 assert(Op1 != Op2 && "Cannot swap operand with itself.");
3461 unsigned MaxOp = std::max(Op1, Op2);
3462 unsigned MinOp = std::min(Op1, Op2);
3463 MachineOperand MOp1 = MI.getOperand(MinOp);
3464 MachineOperand MOp2 = MI.getOperand(MaxOp);
3465 MI.RemoveOperand(std::max(Op1, Op2));
3466 MI.RemoveOperand(std::min(Op1, Op2));
3468 // If the operands we are swapping are the two at the end (the common case)
3469 // we can just remove both and add them in the opposite order.
3470 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
3471 MI.addOperand(MOp2);
3472 MI.addOperand(MOp1);
3474 // Store all operands in a temporary vector, remove them and re-add in the
3476 SmallVector<MachineOperand, 2> MOps;
3477 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
3478 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
3479 MOps.push_back(MI.getOperand(i));
3480 MI.RemoveOperand(i);
3482 // MOp2 needs to be added next.
3483 MI.addOperand(MOp2);
3484 // Now add the rest.
3485 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
3487 MI.addOperand(MOp1);
3489 MI.addOperand(MOps.back());
3496 // Check if the 'MI' that has the index OpNoForForwarding
3497 // meets the requirement described in the ImmInstrInfo.
3498 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
3499 const ImmInstrInfo &III,
3500 unsigned OpNoForForwarding
3502 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
3503 // would not work pre-RA, we can only do the check post RA.
3504 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3508 // Cannot do the transform if MI isn't summing the operands.
3509 if (!III.IsSummingOperands)
3512 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
3513 if (!III.ZeroIsSpecialOrig)
3516 // We cannot do the transform if the operand we are trying to replace
3517 // isn't the same as the operand the instruction allows.
3518 if (OpNoForForwarding != III.OpNoForForwarding)
3521 // Check if the instruction we are trying to transform really has
3522 // the special zero register as its operand.
3523 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
3524 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
3527 // This machine instruction is convertible if it is,
3528 // 1. summing the operands.
3529 // 2. one of the operands is special zero register.
3530 // 3. the operand we are trying to replace is allowed by the MI.
3534 // Check if the DefMI is the add inst and set the ImmMO and RegMO
3536 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
3537 const ImmInstrInfo &III,
3538 MachineOperand *&ImmMO,
3539 MachineOperand *&RegMO) const {
3540 unsigned Opc = DefMI.getOpcode();
3541 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
3544 assert(DefMI.getNumOperands() >= 3 &&
3545 "Add inst must have at least three operands");
3546 RegMO = &DefMI.getOperand(1);
3547 ImmMO = &DefMI.getOperand(2);
3549 // Before RA, ADDI first operand could be a frame index.
3550 if (!RegMO->isReg())
3553 // This DefMI is elgible for forwarding if it is:
3555 // 2. one of the operands is Imm/CPI/Global.
3556 return isAnImmediateOperand(*ImmMO);
3559 bool PPCInstrInfo::isRegElgibleForForwarding(
3560 const MachineOperand &RegMO, const MachineInstr &DefMI,
3561 const MachineInstr &MI, bool KillDefMI,
3562 bool &IsFwdFeederRegKilled) const {
3565 // z = lfdx 0, x -> z = lfd imm(y)
3566 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
3567 // of "y" between the DEF of "x" and "z".
3568 // The query is only valid post RA.
3569 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3573 Register Reg = RegMO.getReg();
3575 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
3576 MachineBasicBlock::const_reverse_iterator It = MI;
3577 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
3579 for (; It != E; ++It) {
3580 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3582 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3583 IsFwdFeederRegKilled = true;
3584 // Made it to DefMI without encountering a clobber.
3585 if ((&*It) == &DefMI)
3588 assert((&*It) == &DefMI && "DefMI is missing");
3590 // If DefMI also defines the register to be forwarded, we can only forward it
3591 // if DefMI is being erased.
3592 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
3598 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
3599 const MachineInstr &DefMI,
3600 const ImmInstrInfo &III,
3602 int64_t BaseImm) const {
3603 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
3604 if (DefMI.getOpcode() == PPC::ADDItocL) {
3605 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
3606 // However, we know that, it is 16-bit width, and has the alignment of 4.
3607 // Check if the instruction met the requirement.
3608 if (III.ImmMustBeMultipleOf > 4 ||
3609 III.TruncateImmTo || III.ImmWidth != 16)
3612 // Going from XForm to DForm loads means that the displacement needs to be
3613 // not just an immediate but also a multiple of 4, or 16 depending on the
3614 // load. A DForm load cannot be represented if it is a multiple of say 2.
3615 // XForm loads do not have this restriction.
3616 if (ImmMO.isGlobal()) {
3617 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
3618 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
3625 if (ImmMO.isImm()) {
3626 // It is Imm, we need to check if the Imm fit the range.
3627 // Sign-extend to 64-bits.
3628 // DefMI may be folded with another imm form instruction, the result Imm is
3629 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
3630 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
3632 if (Imm % III.ImmMustBeMultipleOf)
3634 if (III.TruncateImmTo)
3635 Imm &= ((1 << III.TruncateImmTo) - 1);
3636 if (III.SignedImm) {
3637 APInt ActualValue(64, Imm, true);
3638 if (!ActualValue.isSignedIntN(III.ImmWidth))
3641 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
3642 if ((uint64_t)Imm > UnsignedMax)
3649 // This ImmMO is forwarded if it meets the requriement describle
3654 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
3655 unsigned OpNoForForwarding,
3656 MachineInstr **KilledDef) const {
3657 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
3658 !DefMI.getOperand(1).isImm())
3661 MachineFunction *MF = MI.getParent()->getParent();
3662 MachineRegisterInfo *MRI = &MF->getRegInfo();
3663 bool PostRA = !MRI->isSSA();
3665 int64_t Immediate = DefMI.getOperand(1).getImm();
3666 // Sign-extend to 64-bits.
3667 int64_t SExtImm = SignExtend64<16>(Immediate);
3669 bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
3670 Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
3672 bool ReplaceWithLI = false;
3673 bool Is64BitLI = false;
3676 unsigned Opc = MI.getOpcode();
3681 // FIXME: Any branches conditional on such a comparison can be made
3682 // unconditional. At this time, this happens too infrequently to be worth
3683 // the implementation effort, but if that ever changes, we could convert
3684 // such a pattern here.
3689 // Doing this post-RA would require dataflow analysis to reliably find uses
3690 // of the CR register set by the compare.
3691 // No need to fixup killed/dead flag since this transformation is only valid
3695 // If a compare-immediate is fed by an immediate and is itself an input of
3696 // an ISEL (the most common case) into a COPY of the correct register.
3697 bool Changed = false;
3698 Register DefReg = MI.getOperand(0).getReg();
3699 int64_t Comparand = MI.getOperand(2).getImm();
3700 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
3701 ? (Comparand | 0xFFFFFFFFFFFF0000)
3704 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
3705 unsigned UseOpc = CompareUseMI.getOpcode();
3706 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
3708 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
3709 Register TrueReg = CompareUseMI.getOperand(1).getReg();
3710 Register FalseReg = CompareUseMI.getOperand(2).getReg();
3711 unsigned RegToCopy =
3712 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
3713 if (RegToCopy == PPC::NoRegister)
3715 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
3716 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
3717 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
3718 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
3719 CompareUseMI.RemoveOperand(3);
3720 CompareUseMI.RemoveOperand(2);
3724 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
3725 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
3726 LLVM_DEBUG(dbgs() << "Is converted to:\n");
3727 // Convert to copy and remove unneeded operands.
3728 CompareUseMI.setDesc(get(PPC::COPY));
3729 CompareUseMI.RemoveOperand(3);
3730 CompareUseMI.RemoveOperand(RegToCopy == TrueReg ? 2 : 1);
3731 CmpIselsConverted++;
3733 LLVM_DEBUG(CompareUseMI.dump());
3737 // This may end up incremented multiple times since this function is called
3738 // during a fixed-point transformation, but it is only meant to indicate the
3739 // presence of this opportunity.
3740 MissedConvertibleImmediateInstrs++;
3744 // Immediate forms - may simply be convertable to an LI.
3747 // Does the sum fit in a 16-bit signed field?
3748 int64_t Addend = MI.getOperand(2).getImm();
3749 if (isInt<16>(Addend + SExtImm)) {
3750 ReplaceWithLI = true;
3751 Is64BitLI = Opc == PPC::ADDI8;
3752 NewImm = Addend + SExtImm;
3758 case PPC::RLDICL_rec:
3759 case PPC::RLDICL_32:
3760 case PPC::RLDICL_32_64: {
3761 // Use APInt's rotate function.
3762 int64_t SH = MI.getOperand(2).getImm();
3763 int64_t MB = MI.getOperand(3).getImm();
3764 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
3766 InVal = InVal.rotl(SH);
3767 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
3769 // Can't replace negative values with an LI as that will sign-extend
3770 // and not clear the left bits. If we're setting the CR bit, we will use
3771 // ANDI_rec which won't sign extend, so that's safe.
3772 if (isUInt<15>(InVal.getSExtValue()) ||
3773 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
3774 ReplaceWithLI = true;
3775 Is64BitLI = Opc != PPC::RLDICL_32;
3776 NewImm = InVal.getSExtValue();
3777 SetCR = Opc == PPC::RLDICL_rec;
3784 case PPC::RLWINM_rec:
3785 case PPC::RLWINM8_rec: {
3786 int64_t SH = MI.getOperand(2).getImm();
3787 int64_t MB = MI.getOperand(3).getImm();
3788 int64_t ME = MI.getOperand(4).getImm();
3789 APInt InVal(32, SExtImm, true);
3790 InVal = InVal.rotl(SH);
3791 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
3793 // Can't replace negative values with an LI as that will sign-extend
3794 // and not clear the left bits. If we're setting the CR bit, we will use
3795 // ANDI_rec which won't sign extend, so that's safe.
3796 bool ValueFits = isUInt<15>(InVal.getSExtValue());
3797 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
3798 isUInt<16>(InVal.getSExtValue()));
3800 ReplaceWithLI = true;
3801 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
3802 NewImm = InVal.getSExtValue();
3803 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
3812 int64_t LogicalImm = MI.getOperand(2).getImm();
3814 if (Opc == PPC::ORI || Opc == PPC::ORI8)
3815 Result = LogicalImm | SExtImm;
3817 Result = LogicalImm ^ SExtImm;
3818 if (isInt<16>(Result)) {
3819 ReplaceWithLI = true;
3820 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
3828 if (ReplaceWithLI) {
3829 // We need to be careful with CR-setting instructions we're replacing.
3831 // We don't know anything about uses when we're out of SSA, so only
3832 // replace if the new immediate will be reproduced.
3833 bool ImmChanged = (SExtImm & NewImm) != NewImm;
3834 if (PostRA && ImmChanged)
3838 // If the defining load-immediate has no other uses, we can just replace
3839 // the immediate with the new immediate.
3840 if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
3841 DefMI.getOperand(1).setImm(NewImm);
3843 // If we're not using the GPR result of the CR-setting instruction, we
3844 // just need to and with zero/non-zero depending on the new immediate.
3845 else if (MRI->use_empty(MI.getOperand(0).getReg())) {
3847 assert(Immediate && "Transformation converted zero to non-zero?");
3850 } else if (ImmChanged)
3855 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3856 LLVM_DEBUG(MI.dump());
3857 LLVM_DEBUG(dbgs() << "Fed by:\n");
3858 LLVM_DEBUG(DefMI.dump());
3859 LoadImmediateInfo LII;
3861 LII.Is64Bit = Is64BitLI;
3863 // If we're setting the CR, the original load-immediate must be kept (as an
3864 // operand to ANDI_rec/ANDI8_rec).
3865 if (KilledDef && SetCR)
3866 *KilledDef = nullptr;
3867 replaceInstrWithLI(MI, LII);
3869 // Fixup killed/dead flag after transformation.
3871 // ForwardingOperandReg = LI imm1
3872 // y = op2 imm2, ForwardingOperandReg(killed)
3873 if (IsForwardingOperandKilled)
3874 fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg);
3876 LLVM_DEBUG(dbgs() << "With:\n");
3877 LLVM_DEBUG(MI.dump());
3883 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
3884 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
3885 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3886 bool PostRA = !MRI->isSSA();
3887 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
3892 // Only handle load/store.
3893 if (!MI.mayLoadOrStore())
3896 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
3898 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
3899 "MI must have x-form opcode");
3901 // get Imm Form info.
3903 bool IsVFReg = MI.getOperand(0).isReg()
3904 ? isVFRegister(MI.getOperand(0).getReg())
3907 if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
3910 if (!III.IsSummingOperands)
3913 if (OpNoForForwarding != III.OpNoForForwarding)
3916 MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
3917 if (!ImmOperandMI.isImm())
3921 MachineOperand *ImmMO = nullptr;
3922 MachineOperand *RegMO = nullptr;
3923 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3925 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
3928 // Set ImmBase from imm instruction as base and get new Imm inside
3929 // isImmElgibleForForwarding.
3930 int64_t ImmBase = ImmOperandMI.getImm();
3932 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
3935 // Get killed info in case fixup needed after transformation.
3936 unsigned ForwardKilledOperandReg = ~0U;
3937 if (MI.getOperand(III.OpNoForForwarding).isKill())
3938 ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg();
3941 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3942 LLVM_DEBUG(MI.dump());
3943 LLVM_DEBUG(dbgs() << "Fed by:\n");
3944 LLVM_DEBUG(DefMI.dump());
3946 MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
3947 MI.getOperand(III.OpNoForForwarding).setIsKill(RegMO->isKill());
3948 MI.getOperand(III.ImmOpNo).setImm(Imm);
3950 // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block.
3951 if (DefMI.getParent() == MI.getParent()) {
3952 // Check if reg is killed between MI and DefMI.
3953 auto IsKilledFor = [&](unsigned Reg) {
3954 MachineBasicBlock::const_reverse_iterator It = MI;
3955 MachineBasicBlock::const_reverse_iterator E = DefMI;
3957 for (; It != E; ++It) {
3958 if (It->killsRegister(Reg))
3965 if (RegMO->isKill() || IsKilledFor(RegMO->getReg()))
3966 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
3967 if (ForwardKilledOperandReg != ~0U)
3968 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
3971 LLVM_DEBUG(dbgs() << "With:\n");
3972 LLVM_DEBUG(MI.dump());
3976 // If an X-Form instruction is fed by an add-immediate and one of its operands
3977 // is the literal zero, attempt to forward the source of the add-immediate to
3978 // the corresponding D-Form instruction with the displacement coming from
3979 // the immediate being added.
3980 bool PPCInstrInfo::transformToImmFormFedByAdd(
3981 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
3982 MachineInstr &DefMI, bool KillDefMI) const {
3985 // x = addi reg, imm <----- DefMI
3986 // y = op 0 , x <----- MI
3988 // OpNoForForwarding
3989 // Check if the MI meet the requirement described in the III.
3990 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
3993 // Check if the DefMI meet the requirement
3994 // described in the III. If yes, set the ImmMO and RegMO accordingly.
3995 MachineOperand *ImmMO = nullptr;
3996 MachineOperand *RegMO = nullptr;
3997 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3999 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4001 // As we get the Imm operand now, we need to check if the ImmMO meet
4002 // the requirement described in the III. If yes set the Imm.
4004 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
4007 bool IsFwdFeederRegKilled = false;
4008 // Check if the RegMO can be forwarded to MI.
4009 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
4010 IsFwdFeederRegKilled))
4013 // Get killed info in case fixup needed after transformation.
4014 unsigned ForwardKilledOperandReg = ~0U;
4015 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4016 bool PostRA = !MRI.isSSA();
4017 if (PostRA && MI.getOperand(OpNoForForwarding).isKill())
4018 ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4020 // We know that, the MI and DefMI both meet the pattern, and
4021 // the Imm also meet the requirement with the new Imm-form.
4022 // It is safe to do the transformation now.
4023 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4024 LLVM_DEBUG(MI.dump());
4025 LLVM_DEBUG(dbgs() << "Fed by:\n");
4026 LLVM_DEBUG(DefMI.dump());
4028 // Update the base reg first.
4029 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
4033 // Then, update the imm.
4034 if (ImmMO->isImm()) {
4035 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
4037 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
4040 // Otherwise, it is Constant Pool Index(CPI) or Global,
4041 // which is relocation in fact. We need to replace the special zero
4042 // register with ImmMO.
4043 // Before that, we need to fixup the target flags for imm.
4044 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
4045 if (DefMI.getOpcode() == PPC::ADDItocL)
4046 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
4048 // MI didn't have the interface such as MI.setOperand(i) though
4049 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
4050 // ImmMO, we need to remove ZERO operand and all the operands behind it,
4051 // and, add the ImmMO, then, move back all the operands behind ZERO.
4052 SmallVector<MachineOperand, 2> MOps;
4053 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
4054 MOps.push_back(MI.getOperand(i));
4055 MI.RemoveOperand(i);
4058 // Remove the last MO in the list, which is ZERO operand in fact.
4060 // Add the imm operand.
4061 MI.addOperand(*ImmMO);
4062 // Now add the rest back.
4063 for (auto &MO : MOps)
4067 // Update the opcode.
4068 MI.setDesc(get(III.ImmOpcode));
4070 // Fix up killed/dead flag after transformation.
4072 // x = ADD KilledFwdFeederReg, imm
4073 // n = opn KilledFwdFeederReg(killed), regn
4076 // x = ADD reg(killed), imm
4078 if (IsFwdFeederRegKilled || RegMO->isKill())
4079 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
4081 // ForwardKilledOperandReg = ADD reg, imm
4082 // y = XOP 0, ForwardKilledOperandReg(killed)
4083 if (ForwardKilledOperandReg != ~0U)
4084 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4086 LLVM_DEBUG(dbgs() << "With:\n");
4087 LLVM_DEBUG(MI.dump());
4092 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
4093 const ImmInstrInfo &III,
4094 unsigned ConstantOpNo,
4095 MachineInstr &DefMI) const {
4096 // DefMI must be LI or LI8.
4097 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4098 !DefMI.getOperand(1).isImm())
4101 // Get Imm operand and Sign-extend to 64-bits.
4102 int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm());
4104 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4105 bool PostRA = !MRI.isSSA();
4106 // Exit early if we can't convert this.
4107 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
4109 if (Imm % III.ImmMustBeMultipleOf)
4111 if (III.TruncateImmTo)
4112 Imm &= ((1 << III.TruncateImmTo) - 1);
4113 if (III.SignedImm) {
4114 APInt ActualValue(64, Imm, true);
4115 if (!ActualValue.isSignedIntN(III.ImmWidth))
4118 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
4119 if ((uint64_t)Imm > UnsignedMax)
4123 // If we're post-RA, the instructions don't agree on whether register zero is
4124 // special, we can transform this as long as the register operand that will
4125 // end up in the location where zero is special isn't R0.
4126 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4127 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
4128 III.ZeroIsSpecialNew + 1;
4129 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
4130 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4131 // If R0 is in the operand where zero is special for the new instruction,
4132 // it is unsafe to transform if the constant operand isn't that operand.
4133 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
4134 ConstantOpNo != III.ZeroIsSpecialNew)
4136 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
4137 ConstantOpNo != PosForOrigZero)
4141 // Get killed info in case fixup needed after transformation.
4142 unsigned ForwardKilledOperandReg = ~0U;
4143 if (PostRA && MI.getOperand(ConstantOpNo).isKill())
4144 ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg();
4146 unsigned Opc = MI.getOpcode();
4147 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
4148 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
4149 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
4150 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
4151 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
4152 Opc == PPC::SRD || Opc == PPC::SRD_rec;
4153 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
4154 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
4155 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
4156 Opc == PPC::SRD_rec;
4158 MI.setDesc(get(III.ImmOpcode));
4159 if (ConstantOpNo == III.OpNoForForwarding) {
4160 // Converting shifts to immediate form is a bit tricky since they may do
4161 // one of three things:
4162 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
4163 // 2. If the shift amount is zero, the result is unchanged (save for maybe
4165 // 3. If the shift amount is in [1, OpSize), it's just a shift
4166 if (SpecialShift32 || SpecialShift64) {
4167 LoadImmediateInfo LII;
4170 LII.Is64Bit = SpecialShift64;
4171 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
4172 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
4173 replaceInstrWithLI(MI, LII);
4174 // Shifts by zero don't change the value. If we don't need to set CR0,
4175 // just convert this to a COPY. Can't do this post-RA since we've already
4176 // cleaned up the copies.
4177 else if (!SetCR && ShAmt == 0 && !PostRA) {
4178 MI.RemoveOperand(2);
4179 MI.setDesc(get(PPC::COPY));
4181 // The 32 bit and 64 bit instructions are quite different.
4182 if (SpecialShift32) {
4183 // Left shifts use (N, 0, 31-N).
4184 // Right shifts use (32-N, N, 31) if 0 < N < 32.
4185 // use (0, 0, 31) if N == 0.
4186 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
4187 uint64_t MB = RightShift ? ShAmt : 0;
4188 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
4189 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4190 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
4193 // Left shifts use (N, 63-N).
4194 // Right shifts use (64-N, N) if 0 < N < 64.
4195 // use (0, 0) if N == 0.
4196 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
4197 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
4198 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4199 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
4203 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
4205 // Convert commutative instructions (switch the operands and convert the
4206 // desired one to an immediate.
4207 else if (III.IsCommutative) {
4208 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
4209 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
4211 llvm_unreachable("Should have exited early!");
4213 // For instructions for which the constant register replaces a different
4214 // operand than where the immediate goes, we need to swap them.
4215 if (III.OpNoForForwarding != III.ImmOpNo)
4216 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
4218 // If the special R0/X0 register index are different for original instruction
4219 // and new instruction, we need to fix up the register class in new
4221 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4222 if (III.ZeroIsSpecialNew) {
4223 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
4224 // need to fix up register class.
4225 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4226 if (Register::isVirtualRegister(RegToModify)) {
4227 const TargetRegisterClass *NewRC =
4228 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
4229 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
4230 MRI.setRegClass(RegToModify, NewRC);
4235 // Fix up killed/dead flag after transformation.
4237 // ForwardKilledOperandReg = LI imm
4238 // y = XOP reg, ForwardKilledOperandReg(killed)
4239 if (ForwardKilledOperandReg != ~0U)
4240 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4244 const TargetRegisterClass *
4245 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
4246 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
4247 return &PPC::VSRCRegClass;
4251 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
4252 return PPC::getRecordFormOpcode(Opcode);
4255 // This function returns true if the machine instruction
4256 // always outputs a value by sign-extending a 32 bit value,
4257 // i.e. 0 to 31-th bits are same as 32-th bit.
4258 static bool isSignExtendingOp(const MachineInstr &MI) {
4259 int Opcode = MI.getOpcode();
4260 if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS ||
4261 Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec ||
4262 Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA ||
4263 Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 ||
4264 Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 ||
4265 Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX ||
4266 Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU ||
4267 Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
4268 Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 ||
4269 Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX ||
4270 Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB ||
4271 Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH ||
4272 Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 ||
4273 Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW ||
4274 Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 ||
4275 Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 ||
4276 Opcode == PPC::EXTSB8_32_64)
4279 if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33)
4282 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
4283 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
4284 MI.getOperand(3).getImm() > 0 &&
4285 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
4291 // This function returns true if the machine instruction
4292 // always outputs zeros in higher 32 bits.
4293 static bool isZeroExtendingOp(const MachineInstr &MI) {
4294 int Opcode = MI.getOpcode();
4295 // The 16-bit immediate is sign-extended in li/lis.
4296 // If the most significant bit is zero, all higher bits are zero.
4297 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
4298 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
4299 int64_t Imm = MI.getOperand(1).getImm();
4300 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
4304 // We have some variations of rotate-and-mask instructions
4305 // that clear higher 32-bits.
4306 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
4307 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
4308 Opcode == PPC::RLDICL_32_64) &&
4309 MI.getOperand(3).getImm() >= 32)
4312 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
4313 MI.getOperand(3).getImm() >= 32 &&
4314 MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm())
4317 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
4318 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
4319 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
4320 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
4323 // There are other instructions that clear higher 32-bits.
4324 if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec ||
4325 Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec ||
4326 Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 ||
4327 Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec ||
4328 Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec ||
4329 Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW ||
4330 Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec ||
4331 Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI ||
4332 Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI ||
4333 Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX ||
4334 Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX ||
4335 Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX ||
4336 Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ ||
4337 Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX ||
4338 Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 ||
4339 Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 ||
4340 Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 ||
4341 Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 ||
4342 Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
4343 Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec ||
4344 Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec ||
4345 Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec ||
4346 Opcode == PPC::MFVSRWZ)
4352 // This function returns true if the input MachineInstr is a TOC save
4354 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
4355 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
4357 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
4358 unsigned StackOffset = MI.getOperand(1).getImm();
4359 Register StackReg = MI.getOperand(2).getReg();
4360 if (StackReg == PPC::X1 && StackOffset == TOCSaveOffset)
4366 // We limit the max depth to track incoming values of PHIs or binary ops
4367 // (e.g. AND) to avoid excessive cost.
4368 const unsigned MAX_DEPTH = 1;
4371 PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt,
4372 const unsigned Depth) const {
4373 const MachineFunction *MF = MI.getParent()->getParent();
4374 const MachineRegisterInfo *MRI = &MF->getRegInfo();
4376 // If we know this instruction returns sign- or zero-extended result,
4378 if (SignExt ? isSignExtendingOp(MI):
4379 isZeroExtendingOp(MI))
4382 switch (MI.getOpcode()) {
4384 Register SrcReg = MI.getOperand(1).getReg();
4386 // In both ELFv1 and v2 ABI, method parameters and the return value
4387 // are sign- or zero-extended.
4388 if (MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
4389 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
4390 // We check the ZExt/SExt flags for a method parameter.
4391 if (MI.getParent()->getBasicBlock() ==
4392 &MF->getFunction().getEntryBlock()) {
4393 Register VReg = MI.getOperand(0).getReg();
4394 if (MF->getRegInfo().isLiveIn(VReg))
4395 return SignExt ? FuncInfo->isLiveInSExt(VReg) :
4396 FuncInfo->isLiveInZExt(VReg);
4399 // For a method return value, we check the ZExt/SExt flags in attribute.
4400 // We assume the following code sequence for method call.
4401 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
4402 // BL8_NOP @func,...
4403 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
4404 // %5 = COPY %x3; G8RC:%5
4405 if (SrcReg == PPC::X3) {
4406 const MachineBasicBlock *MBB = MI.getParent();
4407 MachineBasicBlock::const_instr_iterator II =
4408 MachineBasicBlock::const_instr_iterator(&MI);
4409 if (II != MBB->instr_begin() &&
4410 (--II)->getOpcode() == PPC::ADJCALLSTACKUP) {
4411 const MachineInstr &CallMI = *(--II);
4412 if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) {
4413 const Function *CalleeFn =
4414 dyn_cast<Function>(CallMI.getOperand(0).getGlobal());
4417 const IntegerType *IntTy =
4418 dyn_cast<IntegerType>(CalleeFn->getReturnType());
4419 const AttributeSet &Attrs =
4420 CalleeFn->getAttributes().getRetAttributes();
4421 if (IntTy && IntTy->getBitWidth() <= 32)
4422 return Attrs.hasAttribute(SignExt ? Attribute::SExt :
4429 // If this is a copy from another register, we recursively check source.
4430 if (!Register::isVirtualRegister(SrcReg))
4432 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4434 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4440 case PPC::ANDIS_rec:
4445 case PPC::ANDI8_rec:
4446 case PPC::ANDIS8_rec:
4451 // logical operation with 16-bit immediate does not change the upper bits.
4452 // So, we track the operand register as we do for register copy.
4453 Register SrcReg = MI.getOperand(1).getReg();
4454 if (!Register::isVirtualRegister(SrcReg))
4456 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4458 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4463 // If all incoming values are sign-/zero-extended,
4464 // the output of OR, ISEL or PHI is also sign-/zero-extended.
4469 if (Depth >= MAX_DEPTH)
4472 // The input registers for PHI are operand 1, 3, ...
4473 // The input registers for others are operand 1 and 2.
4474 unsigned E = 3, D = 1;
4475 if (MI.getOpcode() == PPC::PHI) {
4476 E = MI.getNumOperands();
4480 for (unsigned I = 1; I != E; I += D) {
4481 if (MI.getOperand(I).isReg()) {
4482 Register SrcReg = MI.getOperand(I).getReg();
4483 if (!Register::isVirtualRegister(SrcReg))
4485 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4486 if (SrcMI == NULL || !isSignOrZeroExtended(*SrcMI, SignExt, Depth+1))
4495 // If at least one of the incoming values of an AND is zero extended
4496 // then the output is also zero-extended. If both of the incoming values
4497 // are sign-extended then the output is also sign extended.
4500 if (Depth >= MAX_DEPTH)
4503 assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg());
4505 Register SrcReg1 = MI.getOperand(1).getReg();
4506 Register SrcReg2 = MI.getOperand(2).getReg();
4508 if (!Register::isVirtualRegister(SrcReg1) ||
4509 !Register::isVirtualRegister(SrcReg2))
4512 const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1);
4513 const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2);
4514 if (!MISrc1 || !MISrc2)
4518 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) &&
4519 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4521 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) ||
4522 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4531 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
4532 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
4536 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
4537 MachineInstr *Loop, *EndLoop, *LoopCount;
4538 MachineFunction *MF;
4539 const TargetInstrInfo *TII;
4543 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
4544 MachineInstr *LoopCount)
4545 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
4546 MF(Loop->getParent()->getParent()),
4547 TII(MF->getSubtarget().getInstrInfo()) {
4548 // Inspect the Loop instruction up-front, as it may be deleted when we call
4549 // createTripCountGreaterCondition.
4550 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
4551 TripCount = LoopCount->getOperand(1).getImm();
4556 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4557 // Only ignore the terminator.
4558 return MI == EndLoop;
4562 createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
4563 SmallVectorImpl<MachineOperand> &Cond) override {
4564 if (TripCount == -1) {
4565 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4566 // so we don't need to generate any thing here.
4567 Cond.push_back(MachineOperand::CreateImm(0));
4568 Cond.push_back(MachineOperand::CreateReg(
4569 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
4574 return TripCount > TC;
4577 void setPreheader(MachineBasicBlock *NewPreheader) override {
4578 // Do nothing. We want the LOOP setup instruction to stay in the *old*
4579 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
4582 void adjustTripCount(int TripCountAdjust) override {
4583 // If the loop trip count is a compile-time value, then just change the
4585 if (LoopCount->getOpcode() == PPC::LI8 ||
4586 LoopCount->getOpcode() == PPC::LI) {
4587 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
4588 LoopCount->getOperand(1).setImm(TripCount);
4592 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4593 // so we don't need to generate any thing here.
4596 void disposed() override {
4597 Loop->eraseFromParent();
4598 // Ensure the loop setup instruction is deleted too.
4599 LoopCount->eraseFromParent();
4604 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
4605 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
4606 // We really "analyze" only hardware loops right now.
4607 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
4608 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
4609 if (Preheader == LoopBB)
4610 Preheader = *std::next(LoopBB->pred_begin());
4611 MachineFunction *MF = Preheader->getParent();
4613 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
4614 SmallPtrSet<MachineBasicBlock *, 8> Visited;
4615 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
4616 Register LoopCountReg = LoopInst->getOperand(0).getReg();
4617 MachineRegisterInfo &MRI = MF->getRegInfo();
4618 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
4619 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
4625 MachineInstr *PPCInstrInfo::findLoopInstr(
4626 MachineBasicBlock &PreHeader,
4627 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
4629 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
4631 // The loop set-up instruction should be in preheader
4632 for (auto &I : PreHeader.instrs())
4633 if (I.getOpcode() == LOOPi)
4638 // Return true if get the base operand, byte offset of an instruction and the
4639 // memory width. Width is the size of memory that is being loaded/stored.
4640 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
4641 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
4642 unsigned &Width, const TargetRegisterInfo *TRI) const {
4643 if (!LdSt.mayLoadOrStore())
4646 // Handle only loads/stores with base register followed by immediate offset.
4647 if (LdSt.getNumExplicitOperands() != 3)
4649 if (!LdSt.getOperand(1).isImm() || !LdSt.getOperand(2).isReg())
4652 if (!LdSt.hasOneMemOperand())
4655 Width = (*LdSt.memoperands_begin())->getSize();
4656 Offset = LdSt.getOperand(1).getImm();
4657 BaseReg = &LdSt.getOperand(2);
4661 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
4662 const MachineInstr &MIa, const MachineInstr &MIb) const {
4663 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
4664 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
4666 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
4667 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4670 // Retrieve the base register, offset from the base register and width. Width
4671 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
4672 // base registers are identical, and the offset of a lower memory access +
4673 // the width doesn't overlap the offset of a higher memory access,
4674 // then the memory accesses are different.
4675 const TargetRegisterInfo *TRI = &getRegisterInfo();
4676 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
4677 int64_t OffsetA = 0, OffsetB = 0;
4678 unsigned int WidthA = 0, WidthB = 0;
4679 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
4680 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
4681 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
4682 int LowOffset = std::min(OffsetA, OffsetB);
4683 int HighOffset = std::max(OffsetA, OffsetB);
4684 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4685 if (LowOffset + LowWidth <= HighOffset)