1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
11 //===----------------------------------------------------------------------===//
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/CodeGen/LiveIntervals.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineMemOperand.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/PseudoSourceValue.h"
30 #include "llvm/CodeGen/ScheduleDAG.h"
31 #include "llvm/CodeGen/SlotIndexes.h"
32 #include "llvm/CodeGen/StackMaps.h"
33 #include "llvm/MC/MCAsmInfo.h"
34 #include "llvm/MC/MCInst.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/TargetRegistry.h"
39 #include "llvm/Support/raw_ostream.h"
43 #define DEBUG_TYPE "ppc-instr-info"
45 #define GET_INSTRMAP_INFO
46 #define GET_INSTRINFO_CTOR_DTOR
47 #include "PPCGenInstrInfo.inc"
49 STATISTIC(NumStoreSPILLVSRRCAsVec,
50 "Number of spillvsrrc spilled to stack as vec");
51 STATISTIC(NumStoreSPILLVSRRCAsGpr,
52 "Number of spillvsrrc spilled to stack as gpr");
53 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
54 STATISTIC(CmpIselsConverted,
55 "Number of ISELs that depend on comparison of constants converted");
56 STATISTIC(MissedConvertibleImmediateInstrs,
57 "Number of compare-immediate instructions fed by constants");
58 STATISTIC(NumRcRotatesConvertedToRcAnd,
59 "Number of record-form rotates converted to record-form andi");
62 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
63 cl::desc("Disable analysis for CTR loops"));
65 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
66 cl::desc("Disable compare instruction optimization"), cl::Hidden);
68 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
69 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
73 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
74 cl::desc("Use the old (incorrect) instruction latency calculation"));
76 // Pin the vtable to this file.
77 void PPCInstrInfo::anchor() {}
79 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
80 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
81 /* CatchRetOpcode */ -1,
82 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
83 Subtarget(STI), RI(STI.getTargetMachine()) {}
85 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
86 /// this target when scheduling the DAG.
87 ScheduleHazardRecognizer *
88 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
89 const ScheduleDAG *DAG) const {
91 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
92 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
93 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
94 const InstrItineraryData *II =
95 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
96 return new ScoreboardHazardRecognizer(II, DAG);
99 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
102 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
103 /// to use for this target when scheduling the DAG.
104 ScheduleHazardRecognizer *
105 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
106 const ScheduleDAG *DAG) const {
108 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
110 // FIXME: Leaving this as-is until we have POWER9 scheduling info
111 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
112 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
114 // Most subtargets use a PPC970 recognizer.
115 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
116 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
117 assert(DAG->TII && "No InstrInfo?");
119 return new PPCHazardRecognizer970(*DAG);
122 return new ScoreboardHazardRecognizer(II, DAG);
125 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
126 const MachineInstr &MI,
127 unsigned *PredCost) const {
128 if (!ItinData || UseOldLatencyCalc)
129 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
131 // The default implementation of getInstrLatency calls getStageLatency, but
132 // getStageLatency does not do the right thing for us. While we have
133 // itinerary, most cores are fully pipelined, and so the itineraries only
134 // express the first part of the pipeline, not every stage. Instead, we need
135 // to use the listed output operand cycle number (using operand 0 here, which
138 unsigned Latency = 1;
139 unsigned DefClass = MI.getDesc().getSchedClass();
140 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
141 const MachineOperand &MO = MI.getOperand(i);
142 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
145 int Cycle = ItinData->getOperandCycle(DefClass, i);
149 Latency = std::max(Latency, (unsigned) Cycle);
155 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
156 const MachineInstr &DefMI, unsigned DefIdx,
157 const MachineInstr &UseMI,
158 unsigned UseIdx) const {
159 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
162 if (!DefMI.getParent())
165 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
166 Register Reg = DefMO.getReg();
169 if (Register::isVirtualRegister(Reg)) {
170 const MachineRegisterInfo *MRI =
171 &DefMI.getParent()->getParent()->getRegInfo();
172 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
173 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
175 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
176 PPC::CRBITRCRegClass.contains(Reg);
179 if (UseMI.isBranch() && IsRegCR) {
181 Latency = getInstrLatency(ItinData, DefMI);
183 // On some cores, there is an additional delay between writing to a condition
184 // register, and using it from a branch.
185 unsigned Directive = Subtarget.getCPUDirective();
199 // FIXME: Is this needed for POWER9?
208 /// This is an architecture-specific helper function of reassociateOps.
209 /// Set special operand attributes for new instructions after reassociation.
210 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
211 MachineInstr &OldMI2,
212 MachineInstr &NewMI1,
213 MachineInstr &NewMI2) const {
214 // Propagate FP flags from the original instructions.
215 // But clear poison-generating flags because those may not be valid now.
216 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
217 NewMI1.setFlags(IntersectedFlags);
218 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
219 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
220 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact);
222 NewMI2.setFlags(IntersectedFlags);
223 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap);
224 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap);
225 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact);
228 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
229 uint16_t Flags) const {
231 MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
232 MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
233 MI.clearFlag(MachineInstr::MIFlag::IsExact);
236 // This function does not list all associative and commutative operations, but
237 // only those worth feeding through the machine combiner in an attempt to
238 // reduce the critical path. Mostly, this means floating-point operations,
239 // because they have high latencies(>=5) (compared to other operations, such as
240 // and/or, which are also associative and commutative, but have low latencies).
241 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
242 switch (Inst.getOpcode()) {
270 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
271 Inst.getFlag(MachineInstr::MIFlag::FmNsz);
284 #define InfoArrayIdxFMAInst 0
285 #define InfoArrayIdxFAddInst 1
286 #define InfoArrayIdxFMULInst 2
287 #define InfoArrayIdxAddOpIdx 3
288 #define InfoArrayIdxMULOpIdx 4
289 // Array keeps info for FMA instructions:
290 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
291 // Index 1(InfoArrayIdxFAddInst): ADD instruction assoaicted with FMA;
292 // Index 2(InfoArrayIdxFMULInst): MUL instruction assoaicted with FMA;
293 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
294 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
295 // second MUL operand index is plus 1.
296 static const uint16_t FMAOpIdxInfo[][5] = {
297 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
298 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2},
299 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2},
300 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
301 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2},
302 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1},
303 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1},
304 {PPC::QVFMADDSs, PPC::QVFADDSs, PPC::QVFMULSs, 3, 1},
305 {PPC::QVFMADD, PPC::QVFADD, PPC::QVFMUL, 3, 1}};
307 // Check if an opcode is a FMA instruction. If it is, return the index in array
308 // FMAOpIdxInfo. Otherwise, return -1.
309 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
310 for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++)
311 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
316 // Try to reassociate FMA chains like below:
319 // A = FADD X, Y (Leaf)
320 // B = FMA A, M21, M22 (Prev)
321 // C = FMA B, M31, M32 (Root)
323 // A = FMA X, M21, M22
324 // B = FMA Y, M31, M32
328 // A = FMA X, M11, M12 (Leaf)
329 // B = FMA A, M21, M22 (Prev)
330 // C = FMA B, M31, M32 (Root)
333 // B = FMA X, M21, M22
334 // D = FMA A, M31, M32
337 // breaking the dependency between A and B, allowing FMA to be executed in
338 // parallel (or back-to-back in a pipeline) instead of depending on each other.
339 bool PPCInstrInfo::getFMAPatterns(
341 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
342 MachineBasicBlock *MBB = Root.getParent();
343 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
345 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
346 for (const auto &MO : Instr.explicit_operands())
347 if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg())))
352 auto IsReassociable = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
353 bool IsLeaf, bool IsAdd) {
356 Idx = getFMAOpIdxInfo(Instr.getOpcode());
359 } else if (Instr.getOpcode() !=
360 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())]
361 [InfoArrayIdxFAddInst])
364 // Instruction can be reassociated.
365 // fast math flags may prohibit reassociation.
366 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
367 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
370 // Instruction operands are virtual registers for reassociation.
371 if (!IsAllOpsVirtualReg(Instr))
377 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
379 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
380 MachineInstr *MIAdd = MRI.getUniqueVRegDef(OpAdd.getReg());
381 // If 'add' operand's def is not in current block, don't do ILP related opt.
382 if (!MIAdd || MIAdd->getParent() != MBB)
385 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
386 // as this fma will be changed later.
387 return IsLeaf ? true : MRI.hasOneNonDBGUse(OpAdd.getReg());
390 int16_t AddOpIdx = -1;
391 // Root must be a valid FMA like instruction.
392 if (!IsReassociable(Root, AddOpIdx, false, false))
395 assert((AddOpIdx >= 0) && "add operand index not right!");
397 Register RegB = Root.getOperand(AddOpIdx).getReg();
398 MachineInstr *Prev = MRI.getUniqueVRegDef(RegB);
400 // Prev must be a valid FMA like instruction.
402 if (!IsReassociable(*Prev, AddOpIdx, false, false))
405 assert((AddOpIdx >= 0) && "add operand index not right!");
407 Register RegA = Prev->getOperand(AddOpIdx).getReg();
408 MachineInstr *Leaf = MRI.getUniqueVRegDef(RegA);
410 if (IsReassociable(*Leaf, AddOpIdx, true, false)) {
411 Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
414 if (IsReassociable(*Leaf, AddOpIdx, true, true)) {
415 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
421 bool PPCInstrInfo::getMachineCombinerPatterns(
423 SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
424 // Using the machine combiner in this way is potentially expensive, so
425 // restrict to when aggressive optimizations are desired.
426 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
429 if (getFMAPatterns(Root, Patterns))
432 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
435 void PPCInstrInfo::genAlternativeCodeSequence(
436 MachineInstr &Root, MachineCombinerPattern Pattern,
437 SmallVectorImpl<MachineInstr *> &InsInstrs,
438 SmallVectorImpl<MachineInstr *> &DelInstrs,
439 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
441 case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
442 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
443 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
446 // Reassociate default patterns.
447 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
448 DelInstrs, InstrIdxForVirtReg);
453 // Currently, only handle two patterns REASSOC_XY_AMM_BMM and
454 // REASSOC_XMM_AMM_BMM. See comments for getFMAPatterns.
455 void PPCInstrInfo::reassociateFMA(
456 MachineInstr &Root, MachineCombinerPattern Pattern,
457 SmallVectorImpl<MachineInstr *> &InsInstrs,
458 SmallVectorImpl<MachineInstr *> &DelInstrs,
459 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
460 MachineFunction *MF = Root.getMF();
461 MachineRegisterInfo &MRI = MF->getRegInfo();
462 MachineOperand &OpC = Root.getOperand(0);
463 Register RegC = OpC.getReg();
464 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
465 MRI.constrainRegClass(RegC, RC);
467 unsigned FmaOp = Root.getOpcode();
468 int16_t Idx = getFMAOpIdxInfo(FmaOp);
469 assert(Idx >= 0 && "Root must be a FMA instruction");
471 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
472 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
473 MachineInstr *Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
475 MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
476 uint16_t IntersectedFlags =
477 Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
479 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
481 Reg = Operand.getReg();
482 MRI.constrainRegClass(Reg, RC);
483 KillFlag = Operand.isKill();
486 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
487 Register &MulOp2, bool &MulOp1KillFlag,
488 bool &MulOp2KillFlag) {
489 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
490 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
493 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32;
494 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
495 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false;
497 GetFMAInstrInfo(Root, RegM31, RegM32, KillM31, KillM32);
498 GetFMAInstrInfo(*Prev, RegM21, RegM22, KillM21, KillM22);
500 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
501 GetFMAInstrInfo(*Leaf, RegM11, RegM12, KillM11, KillM12);
502 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
503 } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
504 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
505 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
508 // Create new virtual registers for the new results instead of
509 // recycling legacy ones because the MachineCombiner's computation of the
510 // critical path requires a new register definition rather than an existing
512 Register NewVRA = MRI.createVirtualRegister(RC);
513 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
515 Register NewVRB = MRI.createVirtualRegister(RC);
516 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
519 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
520 NewVRD = MRI.createVirtualRegister(RC);
521 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
524 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
525 Register RegMul1, bool KillRegMul1,
526 Register RegMul2, bool KillRegMul2) {
527 MI->getOperand(AddOpIdx).setReg(RegAdd);
528 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
529 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
530 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
531 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
532 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
535 if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
536 // Create new instructions for insertion.
537 MachineInstrBuilder MINewB =
538 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
539 .addReg(RegX, getKillRegState(KillX))
540 .addReg(RegM21, getKillRegState(KillM21))
541 .addReg(RegM22, getKillRegState(KillM22));
542 MachineInstrBuilder MINewA =
543 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
544 .addReg(RegY, getKillRegState(KillY))
545 .addReg(RegM31, getKillRegState(KillM31))
546 .addReg(RegM32, getKillRegState(KillM32));
547 // If AddOpIdx is not 1, adjust the order.
549 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
550 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
553 MachineInstrBuilder MINewC =
554 BuildMI(*MF, Root.getDebugLoc(),
555 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
556 .addReg(NewVRB, getKillRegState(true))
557 .addReg(NewVRA, getKillRegState(true));
559 // Update flags for newly created instructions.
560 setSpecialOperandAttr(*MINewA, IntersectedFlags);
561 setSpecialOperandAttr(*MINewB, IntersectedFlags);
562 setSpecialOperandAttr(*MINewC, IntersectedFlags);
564 // Record new instructions for insertion.
565 InsInstrs.push_back(MINewA);
566 InsInstrs.push_back(MINewB);
567 InsInstrs.push_back(MINewC);
568 } else if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
569 assert(NewVRD && "new FMA register not created!");
570 // Create new instructions for insertion.
571 MachineInstrBuilder MINewA =
572 BuildMI(*MF, Leaf->getDebugLoc(),
573 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
574 .addReg(RegM11, getKillRegState(KillM11))
575 .addReg(RegM12, getKillRegState(KillM12));
576 MachineInstrBuilder MINewB =
577 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
578 .addReg(RegX, getKillRegState(KillX))
579 .addReg(RegM21, getKillRegState(KillM21))
580 .addReg(RegM22, getKillRegState(KillM22));
581 MachineInstrBuilder MINewD =
582 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
583 .addReg(NewVRA, getKillRegState(true))
584 .addReg(RegM31, getKillRegState(KillM31))
585 .addReg(RegM32, getKillRegState(KillM32));
586 // If AddOpIdx is not 1, adjust the order.
588 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
589 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
593 MachineInstrBuilder MINewC =
594 BuildMI(*MF, Root.getDebugLoc(),
595 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
596 .addReg(NewVRB, getKillRegState(true))
597 .addReg(NewVRD, getKillRegState(true));
599 // Update flags for newly created instructions.
600 setSpecialOperandAttr(*MINewA, IntersectedFlags);
601 setSpecialOperandAttr(*MINewB, IntersectedFlags);
602 setSpecialOperandAttr(*MINewD, IntersectedFlags);
603 setSpecialOperandAttr(*MINewC, IntersectedFlags);
605 // Record new instructions for insertion.
606 InsInstrs.push_back(MINewA);
607 InsInstrs.push_back(MINewB);
608 InsInstrs.push_back(MINewD);
609 InsInstrs.push_back(MINewC);
612 assert(!InsInstrs.empty() &&
613 "Insertion instructions set should not be empty!");
615 // Record old instructions for deletion.
616 DelInstrs.push_back(Leaf);
617 DelInstrs.push_back(Prev);
618 DelInstrs.push_back(&Root);
621 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
622 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
623 Register &SrcReg, Register &DstReg,
624 unsigned &SubIdx) const {
625 switch (MI.getOpcode()) {
626 default: return false;
629 case PPC::EXTSW_32_64:
630 SrcReg = MI.getOperand(1).getReg();
631 DstReg = MI.getOperand(0).getReg();
632 SubIdx = PPC::sub_32;
637 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
638 int &FrameIndex) const {
639 unsigned Opcode = MI.getOpcode();
640 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
641 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
643 if (End != std::find(OpcodesForSpill, End, Opcode)) {
644 // Check for the operands added by addFrameReference (the immediate is the
645 // offset which defaults to 0).
646 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
647 MI.getOperand(2).isFI()) {
648 FrameIndex = MI.getOperand(2).getIndex();
649 return MI.getOperand(0).getReg();
655 // For opcodes with the ReMaterializable flag set, this function is called to
656 // verify the instruction is really rematable.
657 bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
658 AliasAnalysis *AA) const {
659 switch (MI.getOpcode()) {
661 // This function should only be called for opcodes with the ReMaterializable
663 llvm_unreachable("Unknown rematerializable operation!");
670 case PPC::ADDIStocHA:
671 case PPC::ADDIStocHA8:
673 case PPC::LOAD_STACK_GUARD:
677 case PPC::XXLEQVOnes:
681 case PPC::V_SETALLONESB:
682 case PPC::V_SETALLONESH:
683 case PPC::V_SETALLONES:
691 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
692 int &FrameIndex) const {
693 unsigned Opcode = MI.getOpcode();
694 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
695 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
697 if (End != std::find(OpcodesForSpill, End, Opcode)) {
698 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
699 MI.getOperand(2).isFI()) {
700 FrameIndex = MI.getOperand(2).getIndex();
701 return MI.getOperand(0).getReg();
707 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
709 unsigned OpIdx2) const {
710 MachineFunction &MF = *MI.getParent()->getParent();
712 // Normal instructions can be commuted the obvious way.
713 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
714 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
715 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
716 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
717 // changing the relative order of the mask operands might change what happens
718 // to the high-bits of the mask (and, thus, the result).
720 // Cannot commute if it has a non-zero rotate count.
721 if (MI.getOperand(3).getImm() != 0)
724 // If we have a zero rotate count, we have:
726 // Op0 = (Op1 & ~M) | (Op2 & M)
728 // M = mask((ME+1)&31, (MB-1)&31)
729 // Op0 = (Op2 & ~M) | (Op1 & M)
732 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
733 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
734 Register Reg0 = MI.getOperand(0).getReg();
735 Register Reg1 = MI.getOperand(1).getReg();
736 Register Reg2 = MI.getOperand(2).getReg();
737 unsigned SubReg1 = MI.getOperand(1).getSubReg();
738 unsigned SubReg2 = MI.getOperand(2).getSubReg();
739 bool Reg1IsKill = MI.getOperand(1).isKill();
740 bool Reg2IsKill = MI.getOperand(2).isKill();
741 bool ChangeReg0 = false;
742 // If machine instrs are no longer in two-address forms, update
743 // destination register as well.
745 // Must be two address instruction!
746 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
747 "Expecting a two-address instruction!");
748 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
754 unsigned MB = MI.getOperand(4).getImm();
755 unsigned ME = MI.getOperand(5).getImm();
757 // We can't commute a trivial mask (there is no way to represent an all-zero
759 if (MB == 0 && ME == 31)
763 // Create a new instruction.
764 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
765 bool Reg0IsDead = MI.getOperand(0).isDead();
766 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
767 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
768 .addReg(Reg2, getKillRegState(Reg2IsKill))
769 .addReg(Reg1, getKillRegState(Reg1IsKill))
770 .addImm((ME + 1) & 31)
771 .addImm((MB - 1) & 31);
775 MI.getOperand(0).setReg(Reg2);
776 MI.getOperand(0).setSubReg(SubReg2);
778 MI.getOperand(2).setReg(Reg1);
779 MI.getOperand(1).setReg(Reg2);
780 MI.getOperand(2).setSubReg(SubReg1);
781 MI.getOperand(1).setSubReg(SubReg2);
782 MI.getOperand(2).setIsKill(Reg1IsKill);
783 MI.getOperand(1).setIsKill(Reg2IsKill);
785 // Swap the mask around.
786 MI.getOperand(4).setImm((ME + 1) & 31);
787 MI.getOperand(5).setImm((MB - 1) & 31);
791 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
793 unsigned &SrcOpIdx2) const {
794 // For VSX A-Type FMA instructions, it is the first two operands that can be
795 // commuted, however, because the non-encoded tied input operand is listed
796 // first, the operands to swap are actually the second and third.
798 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
800 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
802 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
804 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
807 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
808 MachineBasicBlock::iterator MI) const {
809 // This function is used for scheduling, and the nop wanted here is the type
810 // that terminates dispatch groups on the POWER cores.
811 unsigned Directive = Subtarget.getCPUDirective();
814 default: Opcode = PPC::NOP; break;
815 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
816 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
817 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
818 // FIXME: Update when POWER9 scheduling model is ready.
819 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
823 BuildMI(MBB, MI, DL, get(Opcode));
826 /// Return the noop instruction to use for a noop.
827 void PPCInstrInfo::getNoop(MCInst &NopInst) const {
828 NopInst.setOpcode(PPC::NOP);
832 // Note: If the condition register is set to CTR or CTR8 then this is a
833 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
834 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
835 MachineBasicBlock *&TBB,
836 MachineBasicBlock *&FBB,
837 SmallVectorImpl<MachineOperand> &Cond,
838 bool AllowModify) const {
839 bool isPPC64 = Subtarget.isPPC64();
841 // If the block has no terminators, it just falls into the block after it.
842 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
846 if (!isUnpredicatedTerminator(*I))
850 // If the BB ends with an unconditional branch to the fallthrough BB,
851 // we eliminate the branch instruction.
852 if (I->getOpcode() == PPC::B &&
853 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
854 I->eraseFromParent();
856 // We update iterator after deleting the last branch.
857 I = MBB.getLastNonDebugInstr();
858 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
863 // Get the last instruction in the block.
864 MachineInstr &LastInst = *I;
866 // If there is only one terminator instruction, process it.
867 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
868 if (LastInst.getOpcode() == PPC::B) {
869 if (!LastInst.getOperand(0).isMBB())
871 TBB = LastInst.getOperand(0).getMBB();
873 } else if (LastInst.getOpcode() == PPC::BCC) {
874 if (!LastInst.getOperand(2).isMBB())
876 // Block ends with fall-through condbranch.
877 TBB = LastInst.getOperand(2).getMBB();
878 Cond.push_back(LastInst.getOperand(0));
879 Cond.push_back(LastInst.getOperand(1));
881 } else if (LastInst.getOpcode() == PPC::BC) {
882 if (!LastInst.getOperand(1).isMBB())
884 // Block ends with fall-through condbranch.
885 TBB = LastInst.getOperand(1).getMBB();
886 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
887 Cond.push_back(LastInst.getOperand(0));
889 } else if (LastInst.getOpcode() == PPC::BCn) {
890 if (!LastInst.getOperand(1).isMBB())
892 // Block ends with fall-through condbranch.
893 TBB = LastInst.getOperand(1).getMBB();
894 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
895 Cond.push_back(LastInst.getOperand(0));
897 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
898 LastInst.getOpcode() == PPC::BDNZ) {
899 if (!LastInst.getOperand(0).isMBB())
901 if (DisableCTRLoopAnal)
903 TBB = LastInst.getOperand(0).getMBB();
904 Cond.push_back(MachineOperand::CreateImm(1));
905 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
908 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
909 LastInst.getOpcode() == PPC::BDZ) {
910 if (!LastInst.getOperand(0).isMBB())
912 if (DisableCTRLoopAnal)
914 TBB = LastInst.getOperand(0).getMBB();
915 Cond.push_back(MachineOperand::CreateImm(0));
916 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
921 // Otherwise, don't know what this is.
925 // Get the instruction before it if it's a terminator.
926 MachineInstr &SecondLastInst = *I;
928 // If there are three terminators, we don't know what sort of block this is.
929 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
932 // If the block ends with PPC::B and PPC:BCC, handle it.
933 if (SecondLastInst.getOpcode() == PPC::BCC &&
934 LastInst.getOpcode() == PPC::B) {
935 if (!SecondLastInst.getOperand(2).isMBB() ||
936 !LastInst.getOperand(0).isMBB())
938 TBB = SecondLastInst.getOperand(2).getMBB();
939 Cond.push_back(SecondLastInst.getOperand(0));
940 Cond.push_back(SecondLastInst.getOperand(1));
941 FBB = LastInst.getOperand(0).getMBB();
943 } else if (SecondLastInst.getOpcode() == PPC::BC &&
944 LastInst.getOpcode() == PPC::B) {
945 if (!SecondLastInst.getOperand(1).isMBB() ||
946 !LastInst.getOperand(0).isMBB())
948 TBB = SecondLastInst.getOperand(1).getMBB();
949 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
950 Cond.push_back(SecondLastInst.getOperand(0));
951 FBB = LastInst.getOperand(0).getMBB();
953 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
954 LastInst.getOpcode() == PPC::B) {
955 if (!SecondLastInst.getOperand(1).isMBB() ||
956 !LastInst.getOperand(0).isMBB())
958 TBB = SecondLastInst.getOperand(1).getMBB();
959 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
960 Cond.push_back(SecondLastInst.getOperand(0));
961 FBB = LastInst.getOperand(0).getMBB();
963 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
964 SecondLastInst.getOpcode() == PPC::BDNZ) &&
965 LastInst.getOpcode() == PPC::B) {
966 if (!SecondLastInst.getOperand(0).isMBB() ||
967 !LastInst.getOperand(0).isMBB())
969 if (DisableCTRLoopAnal)
971 TBB = SecondLastInst.getOperand(0).getMBB();
972 Cond.push_back(MachineOperand::CreateImm(1));
973 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
975 FBB = LastInst.getOperand(0).getMBB();
977 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
978 SecondLastInst.getOpcode() == PPC::BDZ) &&
979 LastInst.getOpcode() == PPC::B) {
980 if (!SecondLastInst.getOperand(0).isMBB() ||
981 !LastInst.getOperand(0).isMBB())
983 if (DisableCTRLoopAnal)
985 TBB = SecondLastInst.getOperand(0).getMBB();
986 Cond.push_back(MachineOperand::CreateImm(0));
987 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
989 FBB = LastInst.getOperand(0).getMBB();
993 // If the block ends with two PPC:Bs, handle it. The second one is not
994 // executed, so remove it.
995 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
996 if (!SecondLastInst.getOperand(0).isMBB())
998 TBB = SecondLastInst.getOperand(0).getMBB();
1001 I->eraseFromParent();
1005 // Otherwise, can't handle this.
1009 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1010 int *BytesRemoved) const {
1011 assert(!BytesRemoved && "code size not handled");
1013 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1017 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1018 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1019 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1020 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1023 // Remove the branch.
1024 I->eraseFromParent();
1028 if (I == MBB.begin()) return 1;
1030 if (I->getOpcode() != PPC::BCC &&
1031 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1032 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1033 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1036 // Remove the branch.
1037 I->eraseFromParent();
1041 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1042 MachineBasicBlock *TBB,
1043 MachineBasicBlock *FBB,
1044 ArrayRef<MachineOperand> Cond,
1046 int *BytesAdded) const {
1047 // Shouldn't be a fall through.
1048 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1049 assert((Cond.size() == 2 || Cond.size() == 0) &&
1050 "PPC branch conditions have two components!");
1051 assert(!BytesAdded && "code size not handled");
1053 bool isPPC64 = Subtarget.isPPC64();
1057 if (Cond.empty()) // Unconditional branch
1058 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1059 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1060 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1061 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1062 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1063 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1064 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1065 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1066 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1067 else // Conditional branch
1068 BuildMI(&MBB, DL, get(PPC::BCC))
1069 .addImm(Cond[0].getImm())
1075 // Two-way Conditional Branch.
1076 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1077 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1078 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1079 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1080 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1081 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1082 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1083 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1085 BuildMI(&MBB, DL, get(PPC::BCC))
1086 .addImm(Cond[0].getImm())
1089 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1094 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1095 ArrayRef<MachineOperand> Cond,
1096 Register DstReg, Register TrueReg,
1097 Register FalseReg, int &CondCycles,
1098 int &TrueCycles, int &FalseCycles) const {
1099 if (Cond.size() != 2)
1102 // If this is really a bdnz-like condition, then it cannot be turned into a
1104 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1107 // Check register classes.
1108 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1109 const TargetRegisterClass *RC =
1110 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1114 // isel is for regular integer GPRs only.
1115 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1116 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1117 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1118 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1121 // FIXME: These numbers are for the A2, how well they work for other cores is
1122 // an open question. On the A2, the isel instruction has a 2-cycle latency
1123 // but single-cycle throughput. These numbers are used in combination with
1124 // the MispredictPenalty setting from the active SchedMachineModel.
1132 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1133 MachineBasicBlock::iterator MI,
1134 const DebugLoc &dl, Register DestReg,
1135 ArrayRef<MachineOperand> Cond, Register TrueReg,
1136 Register FalseReg) const {
1137 assert(Cond.size() == 2 &&
1138 "PPC branch conditions have two components!");
1140 // Get the register classes.
1141 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1142 const TargetRegisterClass *RC =
1143 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1144 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1146 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1147 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1149 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1150 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1151 "isel is for regular integer GPRs only");
1153 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1154 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1156 unsigned SubIdx = 0;
1157 bool SwapOps = false;
1158 switch (SelectPred) {
1160 case PPC::PRED_EQ_MINUS:
1161 case PPC::PRED_EQ_PLUS:
1162 SubIdx = PPC::sub_eq; SwapOps = false; break;
1164 case PPC::PRED_NE_MINUS:
1165 case PPC::PRED_NE_PLUS:
1166 SubIdx = PPC::sub_eq; SwapOps = true; break;
1168 case PPC::PRED_LT_MINUS:
1169 case PPC::PRED_LT_PLUS:
1170 SubIdx = PPC::sub_lt; SwapOps = false; break;
1172 case PPC::PRED_GE_MINUS:
1173 case PPC::PRED_GE_PLUS:
1174 SubIdx = PPC::sub_lt; SwapOps = true; break;
1176 case PPC::PRED_GT_MINUS:
1177 case PPC::PRED_GT_PLUS:
1178 SubIdx = PPC::sub_gt; SwapOps = false; break;
1180 case PPC::PRED_LE_MINUS:
1181 case PPC::PRED_LE_PLUS:
1182 SubIdx = PPC::sub_gt; SwapOps = true; break;
1184 case PPC::PRED_UN_MINUS:
1185 case PPC::PRED_UN_PLUS:
1186 SubIdx = PPC::sub_un; SwapOps = false; break;
1188 case PPC::PRED_NU_MINUS:
1189 case PPC::PRED_NU_PLUS:
1190 SubIdx = PPC::sub_un; SwapOps = true; break;
1191 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1192 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1195 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1196 SecondReg = SwapOps ? TrueReg : FalseReg;
1198 // The first input register of isel cannot be r0. If it is a member
1199 // of a register class that can be r0, then copy it first (the
1200 // register allocator should eliminate the copy).
1201 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1202 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1203 const TargetRegisterClass *FirstRC =
1204 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1205 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1206 Register OldFirstReg = FirstReg;
1207 FirstReg = MRI.createVirtualRegister(FirstRC);
1208 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1209 .addReg(OldFirstReg);
1212 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1213 .addReg(FirstReg).addReg(SecondReg)
1214 .addReg(Cond[1].getReg(), 0, SubIdx);
1217 static unsigned getCRBitValue(unsigned CRBit) {
1219 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1220 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1221 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1222 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1224 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1225 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1226 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1227 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1229 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1230 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1231 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1232 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1234 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1235 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1236 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1237 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1240 assert(Ret != 4 && "Invalid CR bit register");
1244 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1245 MachineBasicBlock::iterator I,
1246 const DebugLoc &DL, MCRegister DestReg,
1247 MCRegister SrcReg, bool KillSrc) const {
1248 // We can end up with self copies and similar things as a result of VSX copy
1249 // legalization. Promote them here.
1250 const TargetRegisterInfo *TRI = &getRegisterInfo();
1251 if (PPC::F8RCRegClass.contains(DestReg) &&
1252 PPC::VSRCRegClass.contains(SrcReg)) {
1253 MCRegister SuperReg =
1254 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1256 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1257 llvm_unreachable("nop VSX copy");
1260 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1261 PPC::VSRCRegClass.contains(DestReg)) {
1262 MCRegister SuperReg =
1263 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1265 if (VSXSelfCopyCrash && DestReg == SuperReg)
1266 llvm_unreachable("nop VSX copy");
1271 // Different class register copy
1272 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1273 PPC::GPRCRegClass.contains(DestReg)) {
1274 MCRegister CRReg = getCRFromCRBit(SrcReg);
1275 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1276 getKillRegState(KillSrc);
1277 // Rotate the CR bit in the CR fields to be the least significant bit and
1278 // then mask with 0x1 (MB = ME = 31).
1279 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1280 .addReg(DestReg, RegState::Kill)
1281 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1285 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1286 PPC::G8RCRegClass.contains(DestReg)) {
1287 BuildMI(MBB, I, DL, get(PPC::MFOCRF8), DestReg).addReg(SrcReg);
1288 getKillRegState(KillSrc);
1290 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1291 PPC::GPRCRegClass.contains(DestReg)) {
1292 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(SrcReg);
1293 getKillRegState(KillSrc);
1295 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1296 PPC::VSFRCRegClass.contains(DestReg)) {
1297 assert(Subtarget.hasDirectMove() &&
1298 "Subtarget doesn't support directmove, don't know how to copy.");
1299 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1301 getKillRegState(KillSrc);
1303 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1304 PPC::G8RCRegClass.contains(DestReg)) {
1305 assert(Subtarget.hasDirectMove() &&
1306 "Subtarget doesn't support directmove, don't know how to copy.");
1307 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1308 getKillRegState(KillSrc);
1310 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1311 PPC::GPRCRegClass.contains(DestReg)) {
1312 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1313 getKillRegState(KillSrc);
1315 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1316 PPC::SPERCRegClass.contains(DestReg)) {
1317 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1318 getKillRegState(KillSrc);
1323 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1325 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1327 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1329 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1331 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1333 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1334 // There are two different ways this can be done:
1335 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1336 // issue in VSU pipeline 0.
1337 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1338 // can go to either pipeline.
1339 // We'll always use xxlor here, because in practically all cases where
1340 // copies are generated, they are close enough to some use that the
1341 // lower-latency form is preferable.
1343 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1344 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1345 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1346 else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
1348 else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
1350 else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
1352 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1354 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1357 llvm_unreachable("Impossible reg-to-reg copy");
1359 const MCInstrDesc &MCID = get(Opc);
1360 if (MCID.getNumOperands() == 3)
1361 BuildMI(MBB, I, DL, MCID, DestReg)
1362 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1364 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1367 static unsigned getSpillIndex(const TargetRegisterClass *RC) {
1368 int OpcodeIndex = 0;
1370 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1371 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1372 OpcodeIndex = SOK_Int4Spill;
1373 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1374 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1375 OpcodeIndex = SOK_Int8Spill;
1376 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1377 OpcodeIndex = SOK_Float8Spill;
1378 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1379 OpcodeIndex = SOK_Float4Spill;
1380 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1381 OpcodeIndex = SOK_SPESpill;
1382 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1383 OpcodeIndex = SOK_CRSpill;
1384 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1385 OpcodeIndex = SOK_CRBitSpill;
1386 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1387 OpcodeIndex = SOK_VRVectorSpill;
1388 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1389 OpcodeIndex = SOK_VSXVectorSpill;
1390 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1391 OpcodeIndex = SOK_VectorFloat8Spill;
1392 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1393 OpcodeIndex = SOK_VectorFloat4Spill;
1394 } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
1395 OpcodeIndex = SOK_VRSaveSpill;
1396 } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
1397 OpcodeIndex = SOK_QuadFloat8Spill;
1398 } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
1399 OpcodeIndex = SOK_QuadFloat4Spill;
1400 } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
1401 OpcodeIndex = SOK_QuadBitSpill;
1402 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1403 OpcodeIndex = SOK_SpillToVSR;
1405 llvm_unreachable("Unknown regclass!");
1411 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1412 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1413 return OpcodesForSpill[getSpillIndex(RC)];
1417 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1418 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1419 return OpcodesForSpill[getSpillIndex(RC)];
1422 void PPCInstrInfo::StoreRegToStackSlot(
1423 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1424 const TargetRegisterClass *RC,
1425 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1426 unsigned Opcode = getStoreOpcodeForSpill(RC);
1429 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1430 FuncInfo->setHasSpills();
1432 NewMIs.push_back(addFrameReference(
1433 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1436 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1437 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1438 FuncInfo->setSpillsCR();
1440 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1441 FuncInfo->setSpillsVRSAVE();
1443 if (isXFormMemOp(Opcode))
1444 FuncInfo->setHasNonRISpills();
1447 void PPCInstrInfo::storeRegToStackSlotNoUpd(
1448 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
1449 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1450 const TargetRegisterInfo *TRI) const {
1451 MachineFunction &MF = *MBB.getParent();
1452 SmallVector<MachineInstr *, 4> NewMIs;
1454 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1456 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1457 MBB.insert(MI, NewMIs[i]);
1459 const MachineFrameInfo &MFI = MF.getFrameInfo();
1460 MachineMemOperand *MMO = MF.getMachineMemOperand(
1461 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1462 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1463 MFI.getObjectAlign(FrameIdx));
1464 NewMIs.back()->addMemOperand(MF, MMO);
1467 void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1468 MachineBasicBlock::iterator MI,
1469 Register SrcReg, bool isKill,
1471 const TargetRegisterClass *RC,
1472 const TargetRegisterInfo *TRI) const {
1473 // We need to avoid a situation in which the value from a VRRC register is
1474 // spilled using an Altivec instruction and reloaded into a VSRC register
1475 // using a VSX instruction. The issue with this is that the VSX
1476 // load/store instructions swap the doublewords in the vector and the Altivec
1477 // ones don't. The register classes on the spill/reload may be different if
1478 // the register is defined using an Altivec instruction and is then used by a
1481 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1484 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1485 unsigned DestReg, int FrameIdx,
1486 const TargetRegisterClass *RC,
1487 SmallVectorImpl<MachineInstr *> &NewMIs)
1489 unsigned Opcode = getLoadOpcodeForSpill(RC);
1490 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1492 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1494 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1495 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1496 FuncInfo->setSpillsCR();
1498 if (PPC::VRSAVERCRegClass.hasSubClassEq(RC))
1499 FuncInfo->setSpillsVRSAVE();
1501 if (isXFormMemOp(Opcode))
1502 FuncInfo->setHasNonRISpills();
1505 void PPCInstrInfo::loadRegFromStackSlotNoUpd(
1506 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
1507 int FrameIdx, const TargetRegisterClass *RC,
1508 const TargetRegisterInfo *TRI) const {
1509 MachineFunction &MF = *MBB.getParent();
1510 SmallVector<MachineInstr*, 4> NewMIs;
1512 if (MI != MBB.end()) DL = MI->getDebugLoc();
1514 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1515 FuncInfo->setHasSpills();
1517 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
1519 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1520 MBB.insert(MI, NewMIs[i]);
1522 const MachineFrameInfo &MFI = MF.getFrameInfo();
1523 MachineMemOperand *MMO = MF.getMachineMemOperand(
1524 MachinePointerInfo::getFixedStack(MF, FrameIdx),
1525 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
1526 MFI.getObjectAlign(FrameIdx));
1527 NewMIs.back()->addMemOperand(MF, MMO);
1530 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1531 MachineBasicBlock::iterator MI,
1532 Register DestReg, int FrameIdx,
1533 const TargetRegisterClass *RC,
1534 const TargetRegisterInfo *TRI) const {
1535 // We need to avoid a situation in which the value from a VRRC register is
1536 // spilled using an Altivec instruction and reloaded into a VSRC register
1537 // using a VSX instruction. The issue with this is that the VSX
1538 // load/store instructions swap the doublewords in the vector and the Altivec
1539 // ones don't. The register classes on the spill/reload may be different if
1540 // the register is defined using an Altivec instruction and is then used by a
1544 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
1548 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
1549 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
1550 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
1551 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
1553 // Leave the CR# the same, but invert the condition.
1554 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
1558 // For some instructions, it is legal to fold ZERO into the RA register field.
1559 // This function performs that fold by replacing the operand with PPC::ZERO,
1560 // it does not consider whether the load immediate zero is no longer in use.
1561 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1562 Register Reg) const {
1563 // A zero immediate should always be loaded with a single li.
1564 unsigned DefOpc = DefMI.getOpcode();
1565 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
1567 if (!DefMI.getOperand(1).isImm())
1569 if (DefMI.getOperand(1).getImm() != 0)
1572 // Note that we cannot here invert the arguments of an isel in order to fold
1573 // a ZERO into what is presented as the second argument. All we have here
1574 // is the condition bit, and that might come from a CR-logical bit operation.
1576 const MCInstrDesc &UseMCID = UseMI.getDesc();
1578 // Only fold into real machine instructions.
1579 if (UseMCID.isPseudo())
1582 // We need to find which of the User's operands is to be folded, that will be
1583 // the operand that matches the given register ID.
1585 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
1586 if (UseMI.getOperand(UseIdx).isReg() &&
1587 UseMI.getOperand(UseIdx).getReg() == Reg)
1590 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
1591 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
1593 const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
1595 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
1596 // register (which might also be specified as a pointer class kind).
1597 if (UseInfo->isLookupPtrRegClass()) {
1598 if (UseInfo->RegClass /* Kind */ != 1)
1601 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
1602 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
1606 // Make sure this is not tied to an output register (or otherwise
1607 // constrained). This is true for ST?UX registers, for example, which
1608 // are tied to their output registers.
1609 if (UseInfo->Constraints != 0)
1613 if (UseInfo->isLookupPtrRegClass()) {
1614 bool isPPC64 = Subtarget.isPPC64();
1615 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
1617 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
1618 PPC::ZERO8 : PPC::ZERO;
1621 UseMI.getOperand(UseIdx).setReg(ZeroReg);
1625 // Folds zero into instructions which have a load immediate zero as an operand
1626 // but also recognize zero as immediate zero. If the definition of the load
1627 // has no more users it is deleted.
1628 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1629 Register Reg, MachineRegisterInfo *MRI) const {
1630 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
1631 if (MRI->use_nodbg_empty(Reg))
1632 DefMI.eraseFromParent();
1636 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
1637 for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
1639 if (I->definesRegister(PPC::CTR) || I->definesRegister(PPC::CTR8))
1644 // We should make sure that, if we're going to predicate both sides of a
1645 // condition (a diamond), that both sides don't define the counter register. We
1646 // can predicate counter-decrement-based branches, but while that predicates
1647 // the branching, it does not predicate the counter decrement. If we tried to
1648 // merge the triangle into one predicated block, we'd decrement the counter
1650 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
1651 unsigned NumT, unsigned ExtraT,
1652 MachineBasicBlock &FMBB,
1653 unsigned NumF, unsigned ExtraF,
1654 BranchProbability Probability) const {
1655 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
1659 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
1660 // The predicated branches are identified by their type, not really by the
1661 // explicit presence of a predicate. Furthermore, some of them can be
1662 // predicated more than once. Because if conversion won't try to predicate
1663 // any instruction which already claims to be predicated (by returning true
1664 // here), always return false. In doing so, we let isPredicable() be the
1665 // final word on whether not the instruction can be (further) predicated.
1670 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
1671 ArrayRef<MachineOperand> Pred) const {
1672 unsigned OpC = MI.getOpcode();
1673 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
1674 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1675 bool isPPC64 = Subtarget.isPPC64();
1676 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
1677 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
1678 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1679 MI.setDesc(get(PPC::BCLR));
1680 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1681 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1682 MI.setDesc(get(PPC::BCLRn));
1683 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1685 MI.setDesc(get(PPC::BCCLR));
1686 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1687 .addImm(Pred[0].getImm())
1692 } else if (OpC == PPC::B) {
1693 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
1694 bool isPPC64 = Subtarget.isPPC64();
1695 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
1696 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
1697 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1698 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1699 MI.RemoveOperand(0);
1701 MI.setDesc(get(PPC::BC));
1702 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1705 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1706 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1707 MI.RemoveOperand(0);
1709 MI.setDesc(get(PPC::BCn));
1710 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1714 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
1715 MI.RemoveOperand(0);
1717 MI.setDesc(get(PPC::BCC));
1718 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1719 .addImm(Pred[0].getImm())
1725 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
1726 OpC == PPC::BCTRL8) {
1727 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
1728 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
1730 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8;
1731 bool isPPC64 = Subtarget.isPPC64();
1733 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
1734 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
1735 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
1736 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1738 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
1739 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
1740 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
1741 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
1745 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
1746 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
1747 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
1748 .addImm(Pred[0].getImm())
1756 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
1757 ArrayRef<MachineOperand> Pred2) const {
1758 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
1759 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
1761 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
1763 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
1766 // P1 can only subsume P2 if they test the same condition register.
1767 if (Pred1[1].getReg() != Pred2[1].getReg())
1770 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
1771 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
1776 // Does P1 subsume P2, e.g. GE subsumes GT.
1777 if (P1 == PPC::PRED_LE &&
1778 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
1780 if (P1 == PPC::PRED_GE &&
1781 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
1787 bool PPCInstrInfo::DefinesPredicate(MachineInstr &MI,
1788 std::vector<MachineOperand> &Pred) const {
1789 // Note: At the present time, the contents of Pred from this function is
1790 // unused by IfConversion. This implementation follows ARM by pushing the
1791 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
1792 // predicate, instructions defining CTR or CTR8 are also included as
1793 // predicate-defining instructions.
1795 const TargetRegisterClass *RCs[] =
1796 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
1797 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
1800 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1801 const MachineOperand &MO = MI.getOperand(i);
1802 for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) {
1803 const TargetRegisterClass *RC = RCs[c];
1805 if (MO.isDef() && RC->contains(MO.getReg())) {
1809 } else if (MO.isRegMask()) {
1810 for (TargetRegisterClass::iterator I = RC->begin(),
1811 IE = RC->end(); I != IE; ++I)
1812 if (MO.clobbersPhysReg(*I)) {
1823 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1824 Register &SrcReg2, int &Mask,
1826 unsigned Opc = MI.getOpcode();
1829 default: return false;
1834 SrcReg = MI.getOperand(1).getReg();
1836 Value = MI.getOperand(2).getImm();
1845 SrcReg = MI.getOperand(1).getReg();
1846 SrcReg2 = MI.getOperand(2).getReg();
1853 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
1854 Register SrcReg2, int Mask, int Value,
1855 const MachineRegisterInfo *MRI) const {
1859 int OpC = CmpInstr.getOpcode();
1860 Register CRReg = CmpInstr.getOperand(0).getReg();
1862 // FP record forms set CR1 based on the exception status bits, not a
1863 // comparison with zero.
1864 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
1867 const TargetRegisterInfo *TRI = &getRegisterInfo();
1868 // The record forms set the condition register based on a signed comparison
1869 // with zero (so says the ISA manual). This is not as straightforward as it
1870 // seems, however, because this is always a 64-bit comparison on PPC64, even
1871 // for instructions that are 32-bit in nature (like slw for example).
1872 // So, on PPC32, for unsigned comparisons, we can use the record forms only
1873 // for equality checks (as those don't depend on the sign). On PPC64,
1874 // we are restricted to equality for unsigned 64-bit comparisons and for
1875 // signed 32-bit comparisons the applicability is more restricted.
1876 bool isPPC64 = Subtarget.isPPC64();
1877 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
1878 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
1879 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
1881 // Look through copies unless that gets us to a physical register.
1882 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
1883 if (ActualSrc.isVirtual())
1886 // Get the unique definition of SrcReg.
1887 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
1888 if (!MI) return false;
1890 bool equalityOnly = false;
1893 if (is32BitSignedCompare) {
1894 // We can perform this optimization only if MI is sign-extending.
1895 if (isSignExtended(*MI))
1899 } else if (is32BitUnsignedCompare) {
1900 // We can perform this optimization, equality only, if MI is
1902 if (isZeroExtended(*MI)) {
1904 equalityOnly = true;
1908 equalityOnly = is64BitUnsignedCompare;
1910 equalityOnly = is32BitUnsignedCompare;
1913 // We need to check the uses of the condition register in order to reject
1914 // non-equality comparisons.
1915 for (MachineRegisterInfo::use_instr_iterator
1916 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
1918 MachineInstr *UseMI = &*I;
1919 if (UseMI->getOpcode() == PPC::BCC) {
1920 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1921 unsigned PredCond = PPC::getPredicateCondition(Pred);
1922 // We ignore hint bits when checking for non-equality comparisons.
1923 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
1925 } else if (UseMI->getOpcode() == PPC::ISEL ||
1926 UseMI->getOpcode() == PPC::ISEL8) {
1927 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
1928 if (SubIdx != PPC::sub_eq)
1935 MachineBasicBlock::iterator I = CmpInstr;
1937 // Scan forward to find the first use of the compare.
1938 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
1940 bool FoundUse = false;
1941 for (MachineRegisterInfo::use_instr_iterator
1942 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
1953 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
1954 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
1956 // There are two possible candidates which can be changed to set CR[01].
1957 // One is MI, the other is a SUB instruction.
1958 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
1959 MachineInstr *Sub = nullptr;
1961 // MI is not a candidate for CMPrr.
1963 // FIXME: Conservatively refuse to convert an instruction which isn't in the
1964 // same BB as the comparison. This is to allow the check below to avoid calls
1965 // (and other explicit clobbers); instead we should really check for these
1966 // more explicitly (in at least a few predecessors).
1967 else if (MI->getParent() != CmpInstr.getParent())
1969 else if (Value != 0) {
1970 // The record-form instructions set CR bit based on signed comparison
1971 // against 0. We try to convert a compare against 1 or -1 into a compare
1972 // against 0 to exploit record-form instructions. For example, we change
1973 // the condition "greater than -1" into "greater than or equal to 0"
1974 // and "less than 1" into "less than or equal to 0".
1976 // Since we optimize comparison based on a specific branch condition,
1977 // we don't optimize if condition code is used by more than once.
1978 if (equalityOnly || !MRI->hasOneUse(CRReg))
1981 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
1982 if (UseMI->getOpcode() != PPC::BCC)
1985 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
1986 unsigned PredCond = PPC::getPredicateCondition(Pred);
1987 unsigned PredHint = PPC::getPredicateHint(Pred);
1988 int16_t Immed = (int16_t)Value;
1990 // When modifying the condition in the predicate, we propagate hint bits
1991 // from the original predicate to the new one.
1992 if (Immed == -1 && PredCond == PPC::PRED_GT)
1993 // We convert "greater than -1" into "greater than or equal to 0",
1994 // since we are assuming signed comparison by !equalityOnly
1995 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
1996 else if (Immed == -1 && PredCond == PPC::PRED_LE)
1997 // We convert "less than or equal to -1" into "less than 0".
1998 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
1999 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2000 // We convert "less than 1" into "less than or equal to 0".
2001 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2002 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2003 // We convert "greater than or equal to 1" into "greater than 0".
2004 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2008 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred));
2014 // Get ready to iterate backward from CmpInstr.
2015 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2017 for (; I != E && !noSub; --I) {
2018 const MachineInstr &Instr = *I;
2019 unsigned IOpC = Instr.getOpcode();
2021 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2022 Instr.readsRegister(PPC::CR0, TRI)))
2023 // This instruction modifies or uses the record condition register after
2024 // the one we want to change. While we could do this transformation, it
2025 // would likely not be profitable. This transformation removes one
2026 // instruction, and so even forcing RA to generate one move probably
2027 // makes it unprofitable.
2030 // Check whether CmpInstr can be made redundant by the current instruction.
2031 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2032 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2033 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2034 ((Instr.getOperand(1).getReg() == SrcReg &&
2035 Instr.getOperand(2).getReg() == SrcReg2) ||
2036 (Instr.getOperand(1).getReg() == SrcReg2 &&
2037 Instr.getOperand(2).getReg() == SrcReg))) {
2043 // The 'and' is below the comparison instruction.
2047 // Return false if no candidates exist.
2051 // The single candidate is called MI.
2055 int MIOpC = MI->getOpcode();
2056 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2057 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2060 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2061 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2065 // FIXME: On the non-embedded POWER architectures, only some of the record
2066 // forms are fast, and we should use only the fast ones.
2068 // The defining instruction has a record form (or is already a record
2069 // form). It is possible, however, that we'll need to reverse the condition
2070 // code of the users.
2074 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2075 // needs to be updated to be based on SUB. Push the condition code
2076 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2077 // condition code of these operands will be modified.
2078 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2079 // comparison against 0, which may modify predicate.
2080 bool ShouldSwap = false;
2081 if (Sub && Value == 0) {
2082 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2083 Sub->getOperand(2).getReg() == SrcReg;
2085 // The operands to subf are the opposite of sub, so only in the fixed-point
2086 // case, invert the order.
2087 ShouldSwap = !ShouldSwap;
2091 for (MachineRegisterInfo::use_instr_iterator
2092 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2094 MachineInstr *UseMI = &*I;
2095 if (UseMI->getOpcode() == PPC::BCC) {
2096 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
2097 unsigned PredCond = PPC::getPredicateCondition(Pred);
2098 assert((!equalityOnly ||
2099 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2100 "Invalid predicate for equality-only optimization");
2101 (void)PredCond; // To suppress warning in release build.
2102 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2103 PPC::getSwappedPredicate(Pred)));
2104 } else if (UseMI->getOpcode() == PPC::ISEL ||
2105 UseMI->getOpcode() == PPC::ISEL8) {
2106 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2107 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2108 "Invalid CR bit for equality-only optimization");
2110 if (NewSubReg == PPC::sub_lt)
2111 NewSubReg = PPC::sub_gt;
2112 else if (NewSubReg == PPC::sub_gt)
2113 NewSubReg = PPC::sub_lt;
2115 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2117 } else // We need to abort on a user we don't understand.
2120 assert(!(Value != 0 && ShouldSwap) &&
2121 "Non-zero immediate support and ShouldSwap"
2122 "may conflict in updating predicate");
2124 // Create a new virtual register to hold the value of the CR set by the
2125 // record-form instruction. If the instruction was not previously in
2126 // record form, then set the kill flag on the CR.
2127 CmpInstr.eraseFromParent();
2129 MachineBasicBlock::iterator MII = MI;
2130 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2131 get(TargetOpcode::COPY), CRReg)
2132 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2134 // Even if CR0 register were dead before, it is alive now since the
2135 // instruction we just built uses it.
2136 MI->clearRegisterDeads(PPC::CR0);
2138 if (MIOpC != NewOpC) {
2139 // We need to be careful here: we're replacing one instruction with
2140 // another, and we need to make sure that we get all of the right
2141 // implicit uses and defs. On the other hand, the caller may be holding
2142 // an iterator to this instruction, and so we can't delete it (this is
2143 // specifically the case if this is the instruction directly after the
2146 // Rotates are expensive instructions. If we're emitting a record-form
2147 // rotate that can just be an andi/andis, we should just emit that.
2148 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2149 Register GPRRes = MI->getOperand(0).getReg();
2150 int64_t SH = MI->getOperand(2).getImm();
2151 int64_t MB = MI->getOperand(3).getImm();
2152 int64_t ME = MI->getOperand(4).getImm();
2153 // We can only do this if both the start and end of the mask are in the
2155 bool MBInLoHWord = MB >= 16;
2156 bool MEInLoHWord = ME >= 16;
2157 uint64_t Mask = ~0LLU;
2159 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2160 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2161 // The mask value needs to shift right 16 if we're emitting andis.
2162 Mask >>= MBInLoHWord ? 0 : 16;
2163 NewOpC = MIOpC == PPC::RLWINM
2164 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2165 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2166 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2167 (ME - MB + 1 == SH) && (MB >= 16)) {
2168 // If we are rotating by the exact number of bits as are in the mask
2169 // and the mask is in the least significant bits of the register,
2170 // that's just an andis. (as long as the GPR result has no uses).
2171 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2173 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2175 // If we've set the mask, we can transform.
2176 if (Mask != ~0LLU) {
2177 MI->RemoveOperand(4);
2178 MI->RemoveOperand(3);
2179 MI->getOperand(2).setImm(Mask);
2180 NumRcRotatesConvertedToRcAnd++;
2182 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2183 int64_t MB = MI->getOperand(3).getImm();
2185 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2186 NewOpC = PPC::ANDI8_rec;
2187 MI->RemoveOperand(3);
2188 MI->getOperand(2).setImm(Mask);
2189 NumRcRotatesConvertedToRcAnd++;
2193 const MCInstrDesc &NewDesc = get(NewOpC);
2194 MI->setDesc(NewDesc);
2196 if (NewDesc.ImplicitDefs)
2197 for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
2198 *ImpDefs; ++ImpDefs)
2199 if (!MI->definesRegister(*ImpDefs))
2200 MI->addOperand(*MI->getParent()->getParent(),
2201 MachineOperand::CreateReg(*ImpDefs, true, true));
2202 if (NewDesc.ImplicitUses)
2203 for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
2204 *ImpUses; ++ImpUses)
2205 if (!MI->readsRegister(*ImpUses))
2206 MI->addOperand(*MI->getParent()->getParent(),
2207 MachineOperand::CreateReg(*ImpUses, false, true));
2209 assert(MI->definesRegister(PPC::CR0) &&
2210 "Record-form instruction does not define cr0?");
2212 // Modify the condition code of operands in OperandsToUpdate.
2213 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2214 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2215 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2216 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2218 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2219 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2224 /// GetInstSize - Return the number of bytes of code the specified
2225 /// instruction may be. This returns the maximum number of bytes.
2227 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2228 unsigned Opcode = MI.getOpcode();
2230 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2231 const MachineFunction *MF = MI.getParent()->getParent();
2232 const char *AsmStr = MI.getOperand(0).getSymbolName();
2233 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2234 } else if (Opcode == TargetOpcode::STACKMAP) {
2235 StackMapOpers Opers(&MI);
2236 return Opers.getNumPatchBytes();
2237 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2238 PatchPointOpers Opers(&MI);
2239 return Opers.getNumPatchBytes();
2241 return get(Opcode).getSize();
2245 std::pair<unsigned, unsigned>
2246 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2247 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2248 return std::make_pair(TF & Mask, TF & ~Mask);
2251 ArrayRef<std::pair<unsigned, const char *>>
2252 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2253 using namespace PPCII;
2254 static const std::pair<unsigned, const char *> TargetFlags[] = {
2257 {MO_TPREL_LO, "ppc-tprel-lo"},
2258 {MO_TPREL_HA, "ppc-tprel-ha"},
2259 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2260 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2261 {MO_TOC_LO, "ppc-toc-lo"},
2262 {MO_TLS, "ppc-tls"}};
2263 return makeArrayRef(TargetFlags);
2266 ArrayRef<std::pair<unsigned, const char *>>
2267 PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
2268 using namespace PPCII;
2269 static const std::pair<unsigned, const char *> TargetFlags[] = {
2270 {MO_PLT, "ppc-plt"},
2271 {MO_PIC_FLAG, "ppc-pic"},
2272 {MO_PCREL_FLAG, "ppc-pcrel"},
2273 {MO_GOT_FLAG, "ppc-got"}};
2274 return makeArrayRef(TargetFlags);
2277 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2278 // The VSX versions have the advantage of a full 64-register target whereas
2279 // the FP ones have the advantage of lower latency and higher throughput. So
2280 // what we are after is using the faster instructions in low register pressure
2281 // situations and using the larger register file in high register pressure
2283 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2284 unsigned UpperOpcode, LowerOpcode;
2285 switch (MI.getOpcode()) {
2286 case PPC::DFLOADf32:
2287 UpperOpcode = PPC::LXSSP;
2288 LowerOpcode = PPC::LFS;
2290 case PPC::DFLOADf64:
2291 UpperOpcode = PPC::LXSD;
2292 LowerOpcode = PPC::LFD;
2294 case PPC::DFSTOREf32:
2295 UpperOpcode = PPC::STXSSP;
2296 LowerOpcode = PPC::STFS;
2298 case PPC::DFSTOREf64:
2299 UpperOpcode = PPC::STXSD;
2300 LowerOpcode = PPC::STFD;
2302 case PPC::XFLOADf32:
2303 UpperOpcode = PPC::LXSSPX;
2304 LowerOpcode = PPC::LFSX;
2306 case PPC::XFLOADf64:
2307 UpperOpcode = PPC::LXSDX;
2308 LowerOpcode = PPC::LFDX;
2310 case PPC::XFSTOREf32:
2311 UpperOpcode = PPC::STXSSPX;
2312 LowerOpcode = PPC::STFSX;
2314 case PPC::XFSTOREf64:
2315 UpperOpcode = PPC::STXSDX;
2316 LowerOpcode = PPC::STFDX;
2319 UpperOpcode = PPC::LXSIWAX;
2320 LowerOpcode = PPC::LFIWAX;
2323 UpperOpcode = PPC::LXSIWZX;
2324 LowerOpcode = PPC::LFIWZX;
2327 UpperOpcode = PPC::STXSIWX;
2328 LowerOpcode = PPC::STFIWX;
2331 llvm_unreachable("Unknown Operation!");
2334 Register TargetReg = MI.getOperand(0).getReg();
2336 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
2337 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
2338 Opcode = LowerOpcode;
2340 Opcode = UpperOpcode;
2341 MI.setDesc(get(Opcode));
2345 static bool isAnImmediateOperand(const MachineOperand &MO) {
2346 return MO.isCPI() || MO.isGlobal() || MO.isImm();
2349 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2350 auto &MBB = *MI.getParent();
2351 auto DL = MI.getDebugLoc();
2353 switch (MI.getOpcode()) {
2354 case TargetOpcode::LOAD_STACK_GUARD: {
2355 assert(Subtarget.isTargetLinux() &&
2356 "Only Linux target is expected to contain LOAD_STACK_GUARD");
2357 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
2358 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
2359 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
2360 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2365 case PPC::DFLOADf32:
2366 case PPC::DFLOADf64:
2367 case PPC::DFSTOREf32:
2368 case PPC::DFSTOREf64: {
2369 assert(Subtarget.hasP9Vector() &&
2370 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
2371 assert(MI.getOperand(2).isReg() &&
2372 isAnImmediateOperand(MI.getOperand(1)) &&
2373 "D-form op must have register and immediate operands");
2374 return expandVSXMemPseudo(MI);
2376 case PPC::XFLOADf32:
2377 case PPC::XFSTOREf32:
2381 assert(Subtarget.hasP8Vector() &&
2382 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
2383 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2384 "X-form op must have register and register operands");
2385 return expandVSXMemPseudo(MI);
2387 case PPC::XFLOADf64:
2388 case PPC::XFSTOREf64: {
2389 assert(Subtarget.hasVSX() &&
2390 "Invalid X-Form Pseudo-ops on target that has no VSX.");
2391 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
2392 "X-form op must have register and register operands");
2393 return expandVSXMemPseudo(MI);
2395 case PPC::SPILLTOVSR_LD: {
2396 Register TargetReg = MI.getOperand(0).getReg();
2397 if (PPC::VSFRCRegClass.contains(TargetReg)) {
2398 MI.setDesc(get(PPC::DFLOADf64));
2399 return expandPostRAPseudo(MI);
2402 MI.setDesc(get(PPC::LD));
2405 case PPC::SPILLTOVSR_ST: {
2406 Register SrcReg = MI.getOperand(0).getReg();
2407 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2408 NumStoreSPILLVSRRCAsVec++;
2409 MI.setDesc(get(PPC::DFSTOREf64));
2410 return expandPostRAPseudo(MI);
2412 NumStoreSPILLVSRRCAsGpr++;
2413 MI.setDesc(get(PPC::STD));
2417 case PPC::SPILLTOVSR_LDX: {
2418 Register TargetReg = MI.getOperand(0).getReg();
2419 if (PPC::VSFRCRegClass.contains(TargetReg))
2420 MI.setDesc(get(PPC::LXSDX));
2422 MI.setDesc(get(PPC::LDX));
2425 case PPC::SPILLTOVSR_STX: {
2426 Register SrcReg = MI.getOperand(0).getReg();
2427 if (PPC::VSFRCRegClass.contains(SrcReg)) {
2428 NumStoreSPILLVSRRCAsVec++;
2429 MI.setDesc(get(PPC::STXSDX));
2431 NumStoreSPILLVSRRCAsGpr++;
2432 MI.setDesc(get(PPC::STDX));
2437 case PPC::CFENCE8: {
2438 auto Val = MI.getOperand(0).getReg();
2439 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
2440 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
2441 .addImm(PPC::PRED_NE_MINUS)
2444 MI.setDesc(get(PPC::ISYNC));
2445 MI.RemoveOperand(0);
2452 // Essentially a compile-time implementation of a compare->isel sequence.
2453 // It takes two constants to compare, along with the true/false registers
2454 // and the comparison type (as a subreg to a CR field) and returns one
2455 // of the true/false registers, depending on the comparison results.
2456 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
2457 unsigned TrueReg, unsigned FalseReg,
2458 unsigned CRSubReg) {
2459 // Signed comparisons. The immediates are assumed to be sign-extended.
2460 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
2462 default: llvm_unreachable("Unknown integer comparison type.");
2464 return Imm1 < Imm2 ? TrueReg : FalseReg;
2466 return Imm1 > Imm2 ? TrueReg : FalseReg;
2468 return Imm1 == Imm2 ? TrueReg : FalseReg;
2471 // Unsigned comparisons.
2472 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
2474 default: llvm_unreachable("Unknown integer comparison type.");
2476 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
2478 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
2480 return Imm1 == Imm2 ? TrueReg : FalseReg;
2483 return PPC::NoRegister;
2486 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
2488 int64_t Imm) const {
2489 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
2490 // Replace the REG with the Immediate.
2491 Register InUseReg = MI.getOperand(OpNo).getReg();
2492 MI.getOperand(OpNo).ChangeToImmediate(Imm);
2494 if (MI.implicit_operands().empty())
2497 // We need to make sure that the MI didn't have any implicit use
2498 // of this REG any more.
2499 const TargetRegisterInfo *TRI = &getRegisterInfo();
2500 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
2501 if (UseOpIdx >= 0) {
2502 MachineOperand &MO = MI.getOperand(UseOpIdx);
2503 if (MO.isImplicit())
2504 // The operands must always be in the following order:
2505 // - explicit reg defs,
2506 // - other explicit operands (reg uses, immediates, etc.),
2507 // - implicit reg defs
2508 // - implicit reg uses
2509 // Therefore, removing the implicit operand won't change the explicit
2511 MI.RemoveOperand(UseOpIdx);
2515 // Replace an instruction with one that materializes a constant (and sets
2516 // CR0 if the original instruction was a record-form instruction).
2517 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
2518 const LoadImmediateInfo &LII) const {
2519 // Remove existing operands.
2520 int OperandToKeep = LII.SetCR ? 1 : 0;
2521 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
2522 MI.RemoveOperand(i);
2524 // Replace the instruction.
2526 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
2527 // Set the immediate.
2528 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2529 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
2533 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
2535 // Set the immediate.
2536 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2540 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
2541 bool &SeenIntermediateUse) const {
2542 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
2543 "Should be called after register allocation.");
2544 const TargetRegisterInfo *TRI = &getRegisterInfo();
2545 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
2547 SeenIntermediateUse = false;
2548 for (; It != E; ++It) {
2549 if (It->modifiesRegister(Reg, TRI))
2551 if (It->readsRegister(Reg, TRI))
2552 SeenIntermediateUse = true;
2557 MachineInstr *PPCInstrInfo::getForwardingDefMI(
2559 unsigned &OpNoForForwarding,
2560 bool &SeenIntermediateUse) const {
2561 OpNoForForwarding = ~0U;
2562 MachineInstr *DefMI = nullptr;
2563 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
2564 const TargetRegisterInfo *TRI = &getRegisterInfo();
2565 // If we're in SSA, get the defs through the MRI. Otherwise, only look
2566 // within the basic block to see if the register is defined using an
2567 // LI/LI8/ADDI/ADDI8.
2569 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2570 if (!MI.getOperand(i).isReg())
2572 Register Reg = MI.getOperand(i).getReg();
2573 if (!Register::isVirtualRegister(Reg))
2575 unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI);
2576 if (Register::isVirtualRegister(TrueReg)) {
2577 DefMI = MRI->getVRegDef(TrueReg);
2578 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
2579 DefMI->getOpcode() == PPC::ADDI ||
2580 DefMI->getOpcode() == PPC::ADDI8) {
2581 OpNoForForwarding = i;
2582 // The ADDI and LI operand maybe exist in one instruction at same
2583 // time. we prefer to fold LI operand as LI only has one Imm operand
2584 // and is more possible to be converted. So if current DefMI is
2585 // ADDI/ADDI8, we continue to find possible LI/LI8.
2586 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
2592 // Looking back through the definition for each operand could be expensive,
2593 // so exit early if this isn't an instruction that either has an immediate
2594 // form or is already an immediate form that we can handle.
2596 unsigned Opc = MI.getOpcode();
2597 bool ConvertibleImmForm =
2598 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
2599 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
2600 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
2601 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
2602 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
2603 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
2604 Opc == PPC::RLWINM8_rec;
2605 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
2606 ? isVFRegister(MI.getOperand(0).getReg())
2608 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
2611 // Don't convert or %X, %Y, %Y since that's just a register move.
2612 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
2613 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
2615 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
2616 MachineOperand &MO = MI.getOperand(i);
2617 SeenIntermediateUse = false;
2618 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
2619 Register Reg = MI.getOperand(i).getReg();
2620 // If we see another use of this reg between the def and the MI,
2621 // we want to flat it so the def isn't deleted.
2622 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
2624 // Is this register defined by some form of add-immediate (including
2625 // load-immediate) within this basic block?
2626 switch (DefMI->getOpcode()) {
2634 OpNoForForwarding = i;
2641 return OpNoForForwarding == ~0U ? nullptr : DefMI;
2644 unsigned PPCInstrInfo::getSpillTarget() const {
2645 return Subtarget.hasP9Vector() ? 1 : 0;
2648 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const {
2649 return StoreSpillOpcodesArray[getSpillTarget()];
2652 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
2653 return LoadSpillOpcodesArray[getSpillTarget()];
2656 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr *StartMI, MachineInstr *EndMI,
2657 unsigned RegNo) const {
2658 // Conservatively clear kill flag for the register if the instructions are in
2659 // different basic blocks and in SSA form, because the kill flag may no longer
2660 // be right. There is no need to bother with dead flags since defs with no
2661 // uses will be handled by DCE.
2662 MachineRegisterInfo &MRI = StartMI->getParent()->getParent()->getRegInfo();
2663 if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) {
2664 MRI.clearKillFlags(RegNo);
2668 // Instructions between [StartMI, EndMI] should be in same basic block.
2669 assert((StartMI->getParent() == EndMI->getParent()) &&
2670 "Instructions are not in same basic block");
2672 // If before RA, StartMI may be def through copy, we need to adjust it to the
2673 // real def. See function getForwardingDefMI.
2674 if (MRI.isSSA() && StartMI->findRegisterUseOperandIdx(RegNo) < 0 &&
2675 StartMI->findRegisterDefOperandIdx(RegNo) < 0) {
2676 assert(Register::isVirtualRegister(RegNo) && "Must be a virtual register");
2677 // Get real def and ignore copies.
2678 StartMI = MRI.getVRegDef(RegNo);
2681 bool IsKillSet = false;
2683 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
2684 MachineOperand &MO = MI.getOperand(Index);
2685 if (MO.isReg() && MO.isUse() && MO.isKill() &&
2686 getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
2687 MO.setIsKill(false);
2690 // Set killed flag for EndMI.
2691 // No need to do anything if EndMI defines RegNo.
2693 EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
2694 if (UseIndex != -1) {
2695 EndMI->getOperand(UseIndex).setIsKill(true);
2697 // Clear killed flag for other EndMI operands related to RegNo. In some
2698 // upexpected cases, killed may be set multiple times for same register
2699 // operand in same MI.
2700 for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i)
2702 clearOperandKillInfo(*EndMI, i);
2705 // Walking the inst in reverse order (EndMI -> StartMI].
2706 MachineBasicBlock::reverse_iterator It = *EndMI;
2707 MachineBasicBlock::reverse_iterator E = EndMI->getParent()->rend();
2708 // EndMI has been handled above, skip it here.
2710 MachineOperand *MO = nullptr;
2711 for (; It != E; ++It) {
2712 // Skip insturctions which could not be a def/use of RegNo.
2713 if (It->isDebugInstr() || It->isPosition())
2716 // Clear killed flag for all It operands related to RegNo. In some
2717 // upexpected cases, killed may be set multiple times for same register
2718 // operand in same MI.
2719 for (int i = 0, e = It->getNumOperands(); i != e; ++i)
2720 clearOperandKillInfo(*It, i);
2722 // If killed is not set, set killed for its last use or set dead for its def
2725 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
2726 // Use found, set it killed.
2728 MO->setIsKill(true);
2730 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
2731 &getRegisterInfo()))) {
2732 // No use found, set dead for its def.
2733 assert(&*It == StartMI && "No new def between StartMI and EndMI.");
2734 MO->setIsDead(true);
2739 if ((&*It) == StartMI)
2742 // Ensure RegMo liveness is killed after EndMI.
2743 assert((IsKillSet || (MO && MO->isDead())) &&
2744 "RegNo should be killed or dead");
2747 // This opt tries to convert the following imm form to an index form to save an
2748 // add for stack variables.
2749 // Return false if no such pattern found.
2751 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2752 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
2753 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
2755 // can be converted to:
2757 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
2758 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
2760 // In order to eliminate ADD instr, make sure that:
2761 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
2762 // new ADDI instr and ADDI can only take int16 Imm.
2763 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
2764 // between ADDI and ADD instr since its original def in ADDI will be changed
2765 // in new ADDI instr. And also there should be no new def for it between
2766 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
2767 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
2768 // between ADD and Imm instr since ADD instr will be eliminated.
2769 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
2770 // moved to Index instr.
2771 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
2772 MachineFunction *MF = MI.getParent()->getParent();
2773 MachineRegisterInfo *MRI = &MF->getRegInfo();
2774 bool PostRA = !MRI->isSSA();
2775 // Do this opt after PEI which is after RA. The reason is stack slot expansion
2776 // in PEI may expose such opportunities since in PEI, stack slot offsets to
2777 // frame base(OffsetAddi) are determined.
2780 unsigned ToBeDeletedReg = 0;
2781 int64_t OffsetImm = 0;
2782 unsigned XFormOpcode = 0;
2785 // Check if Imm instr meets requirement.
2786 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
2790 bool OtherIntermediateUse = false;
2791 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
2793 // Exit if there is other use between ADD and Imm instr or no def found.
2794 if (OtherIntermediateUse || !ADDMI)
2797 // Check if ADD instr meets requirement.
2798 if (!isADDInstrEligibleForFolding(*ADDMI))
2801 unsigned ScaleRegIdx = 0;
2802 int64_t OffsetAddi = 0;
2803 MachineInstr *ADDIMI = nullptr;
2805 // Check if there is a valid ToBeChangedReg in ADDMI.
2806 // 1: It must be killed.
2807 // 2: Its definition must be a valid ADDIMI.
2808 // 3: It must satify int16 offset requirement.
2809 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
2811 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
2816 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
2817 unsigned ToBeChangedReg = ADDIMI->getOperand(0).getReg();
2818 unsigned ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
2819 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
2820 MachineBasicBlock::iterator End) {
2821 for (auto It = ++Start; It != End; It++)
2822 if (It->modifiesRegister(Reg, &getRegisterInfo()))
2827 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
2828 // treated as special zero when ScaleReg is R0/X0 register.
2829 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
2830 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
2833 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
2835 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
2838 // Now start to do the transformation.
2839 LLVM_DEBUG(dbgs() << "Replace instruction: "
2841 LLVM_DEBUG(ADDIMI->dump());
2842 LLVM_DEBUG(ADDMI->dump());
2843 LLVM_DEBUG(MI.dump());
2844 LLVM_DEBUG(dbgs() << "with: "
2847 // Update ADDI instr.
2848 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
2850 // Update Imm instr.
2851 MI.setDesc(get(XFormOpcode));
2852 MI.getOperand(III.ImmOpNo)
2853 .ChangeToRegister(ScaleReg, false, false,
2854 ADDMI->getOperand(ScaleRegIdx).isKill());
2856 MI.getOperand(III.OpNoForForwarding)
2857 .ChangeToRegister(ToBeChangedReg, false, false, true);
2859 // Eliminate ADD instr.
2860 ADDMI->eraseFromParent();
2862 LLVM_DEBUG(ADDIMI->dump());
2863 LLVM_DEBUG(MI.dump());
2868 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
2869 int64_t &Imm) const {
2870 unsigned Opc = ADDIMI.getOpcode();
2872 // Exit if the instruction is not ADDI.
2873 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
2876 // The operand may not necessarily be an immediate - it could be a relocation.
2877 if (!ADDIMI.getOperand(2).isImm())
2880 Imm = ADDIMI.getOperand(2).getImm();
2885 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
2886 unsigned Opc = ADDMI.getOpcode();
2888 // Exit if the instruction is not ADD.
2889 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
2892 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
2893 unsigned &ToBeDeletedReg,
2894 unsigned &XFormOpcode,
2896 ImmInstrInfo &III) const {
2897 // Only handle load/store.
2898 if (!MI.mayLoadOrStore())
2901 unsigned Opc = MI.getOpcode();
2903 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
2905 // Exit if instruction has no index form.
2906 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
2909 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
2910 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
2914 if (!III.IsSummingOperands)
2917 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
2918 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
2919 // Only support imm operands, not relocation slots or others.
2920 if (!ImmOperand.isImm())
2923 assert(RegOperand.isReg() && "Instruction format is not right");
2925 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
2926 if (!RegOperand.isKill())
2929 ToBeDeletedReg = RegOperand.getReg();
2930 OffsetImm = ImmOperand.getImm();
2935 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
2936 MachineInstr *&ADDIMI,
2937 int64_t &OffsetAddi,
2938 int64_t OffsetImm) const {
2939 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
2940 MachineOperand &MO = ADDMI->getOperand(Index);
2945 bool OtherIntermediateUse = false;
2947 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
2948 // Currently handle only one "add + Imminstr" pair case, exit if other
2949 // intermediate use for ToBeChangedReg found.
2950 // TODO: handle the cases where there are other "add + Imminstr" pairs
2951 // with same offset in Imminstr which is like:
2953 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
2954 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
2955 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
2956 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
2957 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
2959 // can be converted to:
2961 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
2962 // (OffsetAddi + OffsetImm)
2963 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
2964 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
2966 if (OtherIntermediateUse || !ADDIMI)
2968 // Check if ADDI instr meets requirement.
2969 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
2972 if (isInt<16>(OffsetAddi + OffsetImm))
2977 // If this instruction has an immediate form and one of its operands is a
2978 // result of a load-immediate or an add-immediate, convert it to
2979 // the immediate form if the constant is in range.
2980 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
2981 MachineInstr **KilledDef) const {
2982 MachineFunction *MF = MI.getParent()->getParent();
2983 MachineRegisterInfo *MRI = &MF->getRegInfo();
2984 bool PostRA = !MRI->isSSA();
2985 bool SeenIntermediateUse = true;
2986 unsigned ForwardingOperand = ~0U;
2987 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
2988 SeenIntermediateUse);
2991 assert(ForwardingOperand < MI.getNumOperands() &&
2992 "The forwarding operand needs to be valid at this point");
2993 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
2994 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
2995 if (KilledDef && KillFwdDefMI)
2998 // If this is a imm instruction and its register operands is produced by ADDI,
2999 // put the imm into imm inst directly.
3000 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3001 PPC::INSTRUCTION_LIST_END &&
3002 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3006 bool IsVFReg = MI.getOperand(0).isReg()
3007 ? isVFRegister(MI.getOperand(0).getReg())
3009 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3010 // If this is a reg+reg instruction that has a reg+imm form,
3011 // and one of the operands is produced by an add-immediate,
3012 // try to convert it.
3014 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3018 // If this is a reg+reg instruction that has a reg+imm form,
3019 // and one of the operands is produced by LI, convert it now.
3021 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3024 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3025 // can be simpified to LI.
3026 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3032 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3033 ImmInstrInfo &III, bool PostRA) const {
3034 // The vast majority of the instructions would need their operand 2 replaced
3035 // with an immediate when switching to the reg+imm form. A marked exception
3036 // are the update form loads/stores for which a constant operand 2 would need
3037 // to turn into a displacement and move operand 1 to the operand 2 position.
3039 III.OpNoForForwarding = 2;
3041 III.ImmMustBeMultipleOf = 1;
3042 III.TruncateImmTo = 0;
3043 III.IsSummingOperands = false;
3045 default: return false;
3048 III.SignedImm = true;
3049 III.ZeroIsSpecialOrig = 0;
3050 III.ZeroIsSpecialNew = 1;
3051 III.IsCommutative = true;
3052 III.IsSummingOperands = true;
3053 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3057 III.SignedImm = true;
3058 III.ZeroIsSpecialOrig = 0;
3059 III.ZeroIsSpecialNew = 0;
3060 III.IsCommutative = true;
3061 III.IsSummingOperands = true;
3062 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3065 III.SignedImm = true;
3066 III.ZeroIsSpecialOrig = 0;
3067 III.ZeroIsSpecialNew = 0;
3068 III.IsCommutative = true;
3069 III.IsSummingOperands = true;
3070 III.ImmOpcode = PPC::ADDIC_rec;
3074 III.SignedImm = true;
3075 III.ZeroIsSpecialOrig = 0;
3076 III.ZeroIsSpecialNew = 0;
3077 III.IsCommutative = false;
3078 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3082 III.SignedImm = true;
3083 III.ZeroIsSpecialOrig = 0;
3084 III.ZeroIsSpecialNew = 0;
3085 III.IsCommutative = false;
3086 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3090 III.SignedImm = false;
3091 III.ZeroIsSpecialOrig = 0;
3092 III.ZeroIsSpecialNew = 0;
3093 III.IsCommutative = false;
3094 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3102 III.SignedImm = false;
3103 III.ZeroIsSpecialOrig = 0;
3104 III.ZeroIsSpecialNew = 0;
3105 III.IsCommutative = true;
3107 default: llvm_unreachable("Unknown opcode");
3109 III.ImmOpcode = PPC::ANDI_rec;
3112 III.ImmOpcode = PPC::ANDI8_rec;
3114 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3115 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3116 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3117 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3122 case PPC::RLWNM_rec:
3123 case PPC::RLWNM8_rec:
3134 III.SignedImm = false;
3135 III.ZeroIsSpecialOrig = 0;
3136 III.ZeroIsSpecialNew = 0;
3137 III.IsCommutative = false;
3138 // This isn't actually true, but the instructions ignore any of the
3139 // upper bits, so any immediate loaded with an LI is acceptable.
3140 // This does not apply to shift right algebraic because a value
3141 // out of range will produce a -1/0.
3143 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3144 Opc == PPC::RLWNM8_rec)
3145 III.TruncateImmTo = 5;
3147 III.TruncateImmTo = 6;
3149 default: llvm_unreachable("Unknown opcode");
3150 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3151 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3152 case PPC::RLWNM_rec:
3153 III.ImmOpcode = PPC::RLWINM_rec;
3155 case PPC::RLWNM8_rec:
3156 III.ImmOpcode = PPC::RLWINM8_rec;
3158 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
3159 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
3161 III.ImmOpcode = PPC::RLWINM_rec;
3164 III.ImmOpcode = PPC::RLWINM8_rec;
3166 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
3167 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
3169 III.ImmOpcode = PPC::RLWINM_rec;
3172 III.ImmOpcode = PPC::RLWINM8_rec;
3176 III.TruncateImmTo = 0;
3177 III.ImmOpcode = PPC::SRAWI;
3181 III.TruncateImmTo = 0;
3182 III.ImmOpcode = PPC::SRAWI_rec;
3187 case PPC::RLDCL_rec:
3189 case PPC::RLDCR_rec:
3196 III.SignedImm = false;
3197 III.ZeroIsSpecialOrig = 0;
3198 III.ZeroIsSpecialNew = 0;
3199 III.IsCommutative = false;
3200 // This isn't actually true, but the instructions ignore any of the
3201 // upper bits, so any immediate loaded with an LI is acceptable.
3202 // This does not apply to shift right algebraic because a value
3203 // out of range will produce a -1/0.
3205 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
3206 Opc == PPC::RLDCR_rec)
3207 III.TruncateImmTo = 6;
3209 III.TruncateImmTo = 7;
3211 default: llvm_unreachable("Unknown opcode");
3212 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
3213 case PPC::RLDCL_rec:
3214 III.ImmOpcode = PPC::RLDICL_rec;
3216 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
3217 case PPC::RLDCR_rec:
3218 III.ImmOpcode = PPC::RLDICR_rec;
3220 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
3222 III.ImmOpcode = PPC::RLDICR_rec;
3224 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
3226 III.ImmOpcode = PPC::RLDICL_rec;
3230 III.TruncateImmTo = 0;
3231 III.ImmOpcode = PPC::SRADI;
3235 III.TruncateImmTo = 0;
3236 III.ImmOpcode = PPC::SRADI_rec;
3240 // Loads and stores:
3262 III.SignedImm = true;
3263 III.ZeroIsSpecialOrig = 1;
3264 III.ZeroIsSpecialNew = 2;
3265 III.IsCommutative = true;
3266 III.IsSummingOperands = true;
3268 III.OpNoForForwarding = 2;
3270 default: llvm_unreachable("Unknown opcode");
3271 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
3272 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
3273 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
3274 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
3275 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
3276 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
3277 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
3278 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
3280 III.ImmOpcode = PPC::LWA;
3281 III.ImmMustBeMultipleOf = 4;
3283 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
3284 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
3285 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
3286 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
3287 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
3288 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
3289 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
3290 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
3291 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
3293 III.ImmOpcode = PPC::STD;
3294 III.ImmMustBeMultipleOf = 4;
3296 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
3297 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
3320 III.SignedImm = true;
3321 III.ZeroIsSpecialOrig = 2;
3322 III.ZeroIsSpecialNew = 3;
3323 III.IsCommutative = false;
3324 III.IsSummingOperands = true;
3326 III.OpNoForForwarding = 3;
3328 default: llvm_unreachable("Unknown opcode");
3329 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
3330 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
3331 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
3332 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
3333 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
3334 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
3335 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
3336 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
3338 III.ImmOpcode = PPC::LDU;
3339 III.ImmMustBeMultipleOf = 4;
3341 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
3342 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
3343 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
3344 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
3345 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
3346 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
3347 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
3348 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
3350 III.ImmOpcode = PPC::STDU;
3351 III.ImmMustBeMultipleOf = 4;
3353 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
3354 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
3357 // Power9 and up only. For some of these, the X-Form version has access to all
3358 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
3359 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
3360 // into or stored from is one of the VR registers.
3367 case PPC::XFLOADf32:
3368 case PPC::XFLOADf64:
3369 case PPC::XFSTOREf32:
3370 case PPC::XFSTOREf64:
3371 if (!Subtarget.hasP9Vector())
3373 III.SignedImm = true;
3374 III.ZeroIsSpecialOrig = 1;
3375 III.ZeroIsSpecialNew = 2;
3376 III.IsCommutative = true;
3377 III.IsSummingOperands = true;
3379 III.OpNoForForwarding = 2;
3380 III.ImmMustBeMultipleOf = 4;
3382 default: llvm_unreachable("Unknown opcode");
3384 III.ImmOpcode = PPC::LXV;
3385 III.ImmMustBeMultipleOf = 16;
3390 III.ImmOpcode = PPC::LXSSP;
3392 III.ImmOpcode = PPC::LFS;
3393 III.ImmMustBeMultipleOf = 1;
3398 case PPC::XFLOADf32:
3399 III.ImmOpcode = PPC::DFLOADf32;
3404 III.ImmOpcode = PPC::LXSD;
3406 III.ImmOpcode = PPC::LFD;
3407 III.ImmMustBeMultipleOf = 1;
3412 case PPC::XFLOADf64:
3413 III.ImmOpcode = PPC::DFLOADf64;
3416 III.ImmOpcode = PPC::STXV;
3417 III.ImmMustBeMultipleOf = 16;
3422 III.ImmOpcode = PPC::STXSSP;
3424 III.ImmOpcode = PPC::STFS;
3425 III.ImmMustBeMultipleOf = 1;
3430 case PPC::XFSTOREf32:
3431 III.ImmOpcode = PPC::DFSTOREf32;
3436 III.ImmOpcode = PPC::STXSD;
3438 III.ImmOpcode = PPC::STFD;
3439 III.ImmMustBeMultipleOf = 1;
3444 case PPC::XFSTOREf64:
3445 III.ImmOpcode = PPC::DFSTOREf64;
3453 // Utility function for swaping two arbitrary operands of an instruction.
3454 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
3455 assert(Op1 != Op2 && "Cannot swap operand with itself.");
3457 unsigned MaxOp = std::max(Op1, Op2);
3458 unsigned MinOp = std::min(Op1, Op2);
3459 MachineOperand MOp1 = MI.getOperand(MinOp);
3460 MachineOperand MOp2 = MI.getOperand(MaxOp);
3461 MI.RemoveOperand(std::max(Op1, Op2));
3462 MI.RemoveOperand(std::min(Op1, Op2));
3464 // If the operands we are swapping are the two at the end (the common case)
3465 // we can just remove both and add them in the opposite order.
3466 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
3467 MI.addOperand(MOp2);
3468 MI.addOperand(MOp1);
3470 // Store all operands in a temporary vector, remove them and re-add in the
3472 SmallVector<MachineOperand, 2> MOps;
3473 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
3474 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
3475 MOps.push_back(MI.getOperand(i));
3476 MI.RemoveOperand(i);
3478 // MOp2 needs to be added next.
3479 MI.addOperand(MOp2);
3480 // Now add the rest.
3481 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
3483 MI.addOperand(MOp1);
3485 MI.addOperand(MOps.back());
3492 // Check if the 'MI' that has the index OpNoForForwarding
3493 // meets the requirement described in the ImmInstrInfo.
3494 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
3495 const ImmInstrInfo &III,
3496 unsigned OpNoForForwarding
3498 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
3499 // would not work pre-RA, we can only do the check post RA.
3500 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3504 // Cannot do the transform if MI isn't summing the operands.
3505 if (!III.IsSummingOperands)
3508 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
3509 if (!III.ZeroIsSpecialOrig)
3512 // We cannot do the transform if the operand we are trying to replace
3513 // isn't the same as the operand the instruction allows.
3514 if (OpNoForForwarding != III.OpNoForForwarding)
3517 // Check if the instruction we are trying to transform really has
3518 // the special zero register as its operand.
3519 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
3520 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
3523 // This machine instruction is convertible if it is,
3524 // 1. summing the operands.
3525 // 2. one of the operands is special zero register.
3526 // 3. the operand we are trying to replace is allowed by the MI.
3530 // Check if the DefMI is the add inst and set the ImmMO and RegMO
3532 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
3533 const ImmInstrInfo &III,
3534 MachineOperand *&ImmMO,
3535 MachineOperand *&RegMO) const {
3536 unsigned Opc = DefMI.getOpcode();
3537 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
3540 assert(DefMI.getNumOperands() >= 3 &&
3541 "Add inst must have at least three operands");
3542 RegMO = &DefMI.getOperand(1);
3543 ImmMO = &DefMI.getOperand(2);
3545 // Before RA, ADDI first operand could be a frame index.
3546 if (!RegMO->isReg())
3549 // This DefMI is elgible for forwarding if it is:
3551 // 2. one of the operands is Imm/CPI/Global.
3552 return isAnImmediateOperand(*ImmMO);
3555 bool PPCInstrInfo::isRegElgibleForForwarding(
3556 const MachineOperand &RegMO, const MachineInstr &DefMI,
3557 const MachineInstr &MI, bool KillDefMI,
3558 bool &IsFwdFeederRegKilled) const {
3561 // z = lfdx 0, x -> z = lfd imm(y)
3562 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
3563 // of "y" between the DEF of "x" and "z".
3564 // The query is only valid post RA.
3565 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3569 Register Reg = RegMO.getReg();
3571 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
3572 MachineBasicBlock::const_reverse_iterator It = MI;
3573 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
3575 for (; It != E; ++It) {
3576 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3578 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
3579 IsFwdFeederRegKilled = true;
3580 // Made it to DefMI without encountering a clobber.
3581 if ((&*It) == &DefMI)
3584 assert((&*It) == &DefMI && "DefMI is missing");
3586 // If DefMI also defines the register to be forwarded, we can only forward it
3587 // if DefMI is being erased.
3588 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
3594 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
3595 const MachineInstr &DefMI,
3596 const ImmInstrInfo &III,
3598 int64_t BaseImm) const {
3599 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
3600 if (DefMI.getOpcode() == PPC::ADDItocL) {
3601 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
3602 // However, we know that, it is 16-bit width, and has the alignment of 4.
3603 // Check if the instruction met the requirement.
3604 if (III.ImmMustBeMultipleOf > 4 ||
3605 III.TruncateImmTo || III.ImmWidth != 16)
3608 // Going from XForm to DForm loads means that the displacement needs to be
3609 // not just an immediate but also a multiple of 4, or 16 depending on the
3610 // load. A DForm load cannot be represented if it is a multiple of say 2.
3611 // XForm loads do not have this restriction.
3612 if (ImmMO.isGlobal()) {
3613 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
3614 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
3621 if (ImmMO.isImm()) {
3622 // It is Imm, we need to check if the Imm fit the range.
3623 // Sign-extend to 64-bits.
3624 // DefMI may be folded with another imm form instruction, the result Imm is
3625 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
3626 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
3628 if (Imm % III.ImmMustBeMultipleOf)
3630 if (III.TruncateImmTo)
3631 Imm &= ((1 << III.TruncateImmTo) - 1);
3632 if (III.SignedImm) {
3633 APInt ActualValue(64, Imm, true);
3634 if (!ActualValue.isSignedIntN(III.ImmWidth))
3637 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
3638 if ((uint64_t)Imm > UnsignedMax)
3645 // This ImmMO is forwarded if it meets the requriement describle
3650 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
3651 unsigned OpNoForForwarding,
3652 MachineInstr **KilledDef) const {
3653 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
3654 !DefMI.getOperand(1).isImm())
3657 MachineFunction *MF = MI.getParent()->getParent();
3658 MachineRegisterInfo *MRI = &MF->getRegInfo();
3659 bool PostRA = !MRI->isSSA();
3661 int64_t Immediate = DefMI.getOperand(1).getImm();
3662 // Sign-extend to 64-bits.
3663 int64_t SExtImm = SignExtend64<16>(Immediate);
3665 bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
3666 Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
3668 bool ReplaceWithLI = false;
3669 bool Is64BitLI = false;
3672 unsigned Opc = MI.getOpcode();
3677 // FIXME: Any branches conditional on such a comparison can be made
3678 // unconditional. At this time, this happens too infrequently to be worth
3679 // the implementation effort, but if that ever changes, we could convert
3680 // such a pattern here.
3685 // Doing this post-RA would require dataflow analysis to reliably find uses
3686 // of the CR register set by the compare.
3687 // No need to fixup killed/dead flag since this transformation is only valid
3691 // If a compare-immediate is fed by an immediate and is itself an input of
3692 // an ISEL (the most common case) into a COPY of the correct register.
3693 bool Changed = false;
3694 Register DefReg = MI.getOperand(0).getReg();
3695 int64_t Comparand = MI.getOperand(2).getImm();
3696 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
3697 ? (Comparand | 0xFFFFFFFFFFFF0000)
3700 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
3701 unsigned UseOpc = CompareUseMI.getOpcode();
3702 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
3704 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
3705 Register TrueReg = CompareUseMI.getOperand(1).getReg();
3706 Register FalseReg = CompareUseMI.getOperand(2).getReg();
3707 unsigned RegToCopy =
3708 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
3709 if (RegToCopy == PPC::NoRegister)
3711 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
3712 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
3713 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
3714 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
3715 CompareUseMI.RemoveOperand(3);
3716 CompareUseMI.RemoveOperand(2);
3720 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
3721 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
3722 LLVM_DEBUG(dbgs() << "Is converted to:\n");
3723 // Convert to copy and remove unneeded operands.
3724 CompareUseMI.setDesc(get(PPC::COPY));
3725 CompareUseMI.RemoveOperand(3);
3726 CompareUseMI.RemoveOperand(RegToCopy == TrueReg ? 2 : 1);
3727 CmpIselsConverted++;
3729 LLVM_DEBUG(CompareUseMI.dump());
3733 // This may end up incremented multiple times since this function is called
3734 // during a fixed-point transformation, but it is only meant to indicate the
3735 // presence of this opportunity.
3736 MissedConvertibleImmediateInstrs++;
3740 // Immediate forms - may simply be convertable to an LI.
3743 // Does the sum fit in a 16-bit signed field?
3744 int64_t Addend = MI.getOperand(2).getImm();
3745 if (isInt<16>(Addend + SExtImm)) {
3746 ReplaceWithLI = true;
3747 Is64BitLI = Opc == PPC::ADDI8;
3748 NewImm = Addend + SExtImm;
3754 case PPC::RLDICL_rec:
3755 case PPC::RLDICL_32:
3756 case PPC::RLDICL_32_64: {
3757 // Use APInt's rotate function.
3758 int64_t SH = MI.getOperand(2).getImm();
3759 int64_t MB = MI.getOperand(3).getImm();
3760 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
3762 InVal = InVal.rotl(SH);
3763 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
3765 // Can't replace negative values with an LI as that will sign-extend
3766 // and not clear the left bits. If we're setting the CR bit, we will use
3767 // ANDI_rec which won't sign extend, so that's safe.
3768 if (isUInt<15>(InVal.getSExtValue()) ||
3769 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
3770 ReplaceWithLI = true;
3771 Is64BitLI = Opc != PPC::RLDICL_32;
3772 NewImm = InVal.getSExtValue();
3773 SetCR = Opc == PPC::RLDICL_rec;
3780 case PPC::RLWINM_rec:
3781 case PPC::RLWINM8_rec: {
3782 int64_t SH = MI.getOperand(2).getImm();
3783 int64_t MB = MI.getOperand(3).getImm();
3784 int64_t ME = MI.getOperand(4).getImm();
3785 APInt InVal(32, SExtImm, true);
3786 InVal = InVal.rotl(SH);
3787 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
3789 // Can't replace negative values with an LI as that will sign-extend
3790 // and not clear the left bits. If we're setting the CR bit, we will use
3791 // ANDI_rec which won't sign extend, so that's safe.
3792 bool ValueFits = isUInt<15>(InVal.getSExtValue());
3793 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
3794 isUInt<16>(InVal.getSExtValue()));
3796 ReplaceWithLI = true;
3797 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
3798 NewImm = InVal.getSExtValue();
3799 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
3808 int64_t LogicalImm = MI.getOperand(2).getImm();
3810 if (Opc == PPC::ORI || Opc == PPC::ORI8)
3811 Result = LogicalImm | SExtImm;
3813 Result = LogicalImm ^ SExtImm;
3814 if (isInt<16>(Result)) {
3815 ReplaceWithLI = true;
3816 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
3824 if (ReplaceWithLI) {
3825 // We need to be careful with CR-setting instructions we're replacing.
3827 // We don't know anything about uses when we're out of SSA, so only
3828 // replace if the new immediate will be reproduced.
3829 bool ImmChanged = (SExtImm & NewImm) != NewImm;
3830 if (PostRA && ImmChanged)
3834 // If the defining load-immediate has no other uses, we can just replace
3835 // the immediate with the new immediate.
3836 if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
3837 DefMI.getOperand(1).setImm(NewImm);
3839 // If we're not using the GPR result of the CR-setting instruction, we
3840 // just need to and with zero/non-zero depending on the new immediate.
3841 else if (MRI->use_empty(MI.getOperand(0).getReg())) {
3843 assert(Immediate && "Transformation converted zero to non-zero?");
3846 } else if (ImmChanged)
3851 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3852 LLVM_DEBUG(MI.dump());
3853 LLVM_DEBUG(dbgs() << "Fed by:\n");
3854 LLVM_DEBUG(DefMI.dump());
3855 LoadImmediateInfo LII;
3857 LII.Is64Bit = Is64BitLI;
3859 // If we're setting the CR, the original load-immediate must be kept (as an
3860 // operand to ANDI_rec/ANDI8_rec).
3861 if (KilledDef && SetCR)
3862 *KilledDef = nullptr;
3863 replaceInstrWithLI(MI, LII);
3865 // Fixup killed/dead flag after transformation.
3867 // ForwardingOperandReg = LI imm1
3868 // y = op2 imm2, ForwardingOperandReg(killed)
3869 if (IsForwardingOperandKilled)
3870 fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg);
3872 LLVM_DEBUG(dbgs() << "With:\n");
3873 LLVM_DEBUG(MI.dump());
3879 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
3880 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
3881 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3882 bool PostRA = !MRI->isSSA();
3883 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
3888 // Only handle load/store.
3889 if (!MI.mayLoadOrStore())
3892 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
3894 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
3895 "MI must have x-form opcode");
3897 // get Imm Form info.
3899 bool IsVFReg = MI.getOperand(0).isReg()
3900 ? isVFRegister(MI.getOperand(0).getReg())
3903 if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
3906 if (!III.IsSummingOperands)
3909 if (OpNoForForwarding != III.OpNoForForwarding)
3912 MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
3913 if (!ImmOperandMI.isImm())
3917 MachineOperand *ImmMO = nullptr;
3918 MachineOperand *RegMO = nullptr;
3919 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3921 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
3924 // Set ImmBase from imm instruction as base and get new Imm inside
3925 // isImmElgibleForForwarding.
3926 int64_t ImmBase = ImmOperandMI.getImm();
3928 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
3931 // Get killed info in case fixup needed after transformation.
3932 unsigned ForwardKilledOperandReg = ~0U;
3933 if (MI.getOperand(III.OpNoForForwarding).isKill())
3934 ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg();
3937 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
3938 LLVM_DEBUG(MI.dump());
3939 LLVM_DEBUG(dbgs() << "Fed by:\n");
3940 LLVM_DEBUG(DefMI.dump());
3942 MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
3943 MI.getOperand(III.OpNoForForwarding).setIsKill(RegMO->isKill());
3944 MI.getOperand(III.ImmOpNo).setImm(Imm);
3946 // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block.
3947 if (DefMI.getParent() == MI.getParent()) {
3948 // Check if reg is killed between MI and DefMI.
3949 auto IsKilledFor = [&](unsigned Reg) {
3950 MachineBasicBlock::const_reverse_iterator It = MI;
3951 MachineBasicBlock::const_reverse_iterator E = DefMI;
3953 for (; It != E; ++It) {
3954 if (It->killsRegister(Reg))
3961 if (RegMO->isKill() || IsKilledFor(RegMO->getReg()))
3962 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
3963 if (ForwardKilledOperandReg != ~0U)
3964 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
3967 LLVM_DEBUG(dbgs() << "With:\n");
3968 LLVM_DEBUG(MI.dump());
3972 // If an X-Form instruction is fed by an add-immediate and one of its operands
3973 // is the literal zero, attempt to forward the source of the add-immediate to
3974 // the corresponding D-Form instruction with the displacement coming from
3975 // the immediate being added.
3976 bool PPCInstrInfo::transformToImmFormFedByAdd(
3977 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
3978 MachineInstr &DefMI, bool KillDefMI) const {
3981 // x = addi reg, imm <----- DefMI
3982 // y = op 0 , x <----- MI
3984 // OpNoForForwarding
3985 // Check if the MI meet the requirement described in the III.
3986 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
3989 // Check if the DefMI meet the requirement
3990 // described in the III. If yes, set the ImmMO and RegMO accordingly.
3991 MachineOperand *ImmMO = nullptr;
3992 MachineOperand *RegMO = nullptr;
3993 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
3995 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
3997 // As we get the Imm operand now, we need to check if the ImmMO meet
3998 // the requirement described in the III. If yes set the Imm.
4000 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
4003 bool IsFwdFeederRegKilled = false;
4004 // Check if the RegMO can be forwarded to MI.
4005 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
4006 IsFwdFeederRegKilled))
4009 // Get killed info in case fixup needed after transformation.
4010 unsigned ForwardKilledOperandReg = ~0U;
4011 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4012 bool PostRA = !MRI.isSSA();
4013 if (PostRA && MI.getOperand(OpNoForForwarding).isKill())
4014 ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4016 // We know that, the MI and DefMI both meet the pattern, and
4017 // the Imm also meet the requirement with the new Imm-form.
4018 // It is safe to do the transformation now.
4019 LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4020 LLVM_DEBUG(MI.dump());
4021 LLVM_DEBUG(dbgs() << "Fed by:\n");
4022 LLVM_DEBUG(DefMI.dump());
4024 // Update the base reg first.
4025 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
4029 // Then, update the imm.
4030 if (ImmMO->isImm()) {
4031 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
4033 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
4036 // Otherwise, it is Constant Pool Index(CPI) or Global,
4037 // which is relocation in fact. We need to replace the special zero
4038 // register with ImmMO.
4039 // Before that, we need to fixup the target flags for imm.
4040 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
4041 if (DefMI.getOpcode() == PPC::ADDItocL)
4042 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
4044 // MI didn't have the interface such as MI.setOperand(i) though
4045 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
4046 // ImmMO, we need to remove ZERO operand and all the operands behind it,
4047 // and, add the ImmMO, then, move back all the operands behind ZERO.
4048 SmallVector<MachineOperand, 2> MOps;
4049 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
4050 MOps.push_back(MI.getOperand(i));
4051 MI.RemoveOperand(i);
4054 // Remove the last MO in the list, which is ZERO operand in fact.
4056 // Add the imm operand.
4057 MI.addOperand(*ImmMO);
4058 // Now add the rest back.
4059 for (auto &MO : MOps)
4063 // Update the opcode.
4064 MI.setDesc(get(III.ImmOpcode));
4066 // Fix up killed/dead flag after transformation.
4068 // x = ADD KilledFwdFeederReg, imm
4069 // n = opn KilledFwdFeederReg(killed), regn
4072 // x = ADD reg(killed), imm
4074 if (IsFwdFeederRegKilled || RegMO->isKill())
4075 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
4077 // ForwardKilledOperandReg = ADD reg, imm
4078 // y = XOP 0, ForwardKilledOperandReg(killed)
4079 if (ForwardKilledOperandReg != ~0U)
4080 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4082 LLVM_DEBUG(dbgs() << "With:\n");
4083 LLVM_DEBUG(MI.dump());
4088 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
4089 const ImmInstrInfo &III,
4090 unsigned ConstantOpNo,
4091 MachineInstr &DefMI) const {
4092 // DefMI must be LI or LI8.
4093 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4094 !DefMI.getOperand(1).isImm())
4097 // Get Imm operand and Sign-extend to 64-bits.
4098 int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm());
4100 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4101 bool PostRA = !MRI.isSSA();
4102 // Exit early if we can't convert this.
4103 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
4105 if (Imm % III.ImmMustBeMultipleOf)
4107 if (III.TruncateImmTo)
4108 Imm &= ((1 << III.TruncateImmTo) - 1);
4109 if (III.SignedImm) {
4110 APInt ActualValue(64, Imm, true);
4111 if (!ActualValue.isSignedIntN(III.ImmWidth))
4114 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
4115 if ((uint64_t)Imm > UnsignedMax)
4119 // If we're post-RA, the instructions don't agree on whether register zero is
4120 // special, we can transform this as long as the register operand that will
4121 // end up in the location where zero is special isn't R0.
4122 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4123 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
4124 III.ZeroIsSpecialNew + 1;
4125 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
4126 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4127 // If R0 is in the operand where zero is special for the new instruction,
4128 // it is unsafe to transform if the constant operand isn't that operand.
4129 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
4130 ConstantOpNo != III.ZeroIsSpecialNew)
4132 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
4133 ConstantOpNo != PosForOrigZero)
4137 // Get killed info in case fixup needed after transformation.
4138 unsigned ForwardKilledOperandReg = ~0U;
4139 if (PostRA && MI.getOperand(ConstantOpNo).isKill())
4140 ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg();
4142 unsigned Opc = MI.getOpcode();
4143 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
4144 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
4145 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
4146 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
4147 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
4148 Opc == PPC::SRD || Opc == PPC::SRD_rec;
4149 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
4150 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
4151 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
4152 Opc == PPC::SRD_rec;
4154 MI.setDesc(get(III.ImmOpcode));
4155 if (ConstantOpNo == III.OpNoForForwarding) {
4156 // Converting shifts to immediate form is a bit tricky since they may do
4157 // one of three things:
4158 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
4159 // 2. If the shift amount is zero, the result is unchanged (save for maybe
4161 // 3. If the shift amount is in [1, OpSize), it's just a shift
4162 if (SpecialShift32 || SpecialShift64) {
4163 LoadImmediateInfo LII;
4166 LII.Is64Bit = SpecialShift64;
4167 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
4168 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
4169 replaceInstrWithLI(MI, LII);
4170 // Shifts by zero don't change the value. If we don't need to set CR0,
4171 // just convert this to a COPY. Can't do this post-RA since we've already
4172 // cleaned up the copies.
4173 else if (!SetCR && ShAmt == 0 && !PostRA) {
4174 MI.RemoveOperand(2);
4175 MI.setDesc(get(PPC::COPY));
4177 // The 32 bit and 64 bit instructions are quite different.
4178 if (SpecialShift32) {
4179 // Left shifts use (N, 0, 31-N).
4180 // Right shifts use (32-N, N, 31) if 0 < N < 32.
4181 // use (0, 0, 31) if N == 0.
4182 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
4183 uint64_t MB = RightShift ? ShAmt : 0;
4184 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
4185 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4186 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
4189 // Left shifts use (N, 63-N).
4190 // Right shifts use (64-N, N) if 0 < N < 64.
4191 // use (0, 0) if N == 0.
4192 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
4193 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
4194 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4195 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
4199 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
4201 // Convert commutative instructions (switch the operands and convert the
4202 // desired one to an immediate.
4203 else if (III.IsCommutative) {
4204 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
4205 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
4207 llvm_unreachable("Should have exited early!");
4209 // For instructions for which the constant register replaces a different
4210 // operand than where the immediate goes, we need to swap them.
4211 if (III.OpNoForForwarding != III.ImmOpNo)
4212 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
4214 // If the special R0/X0 register index are different for original instruction
4215 // and new instruction, we need to fix up the register class in new
4217 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4218 if (III.ZeroIsSpecialNew) {
4219 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
4220 // need to fix up register class.
4221 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4222 if (Register::isVirtualRegister(RegToModify)) {
4223 const TargetRegisterClass *NewRC =
4224 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
4225 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
4226 MRI.setRegClass(RegToModify, NewRC);
4231 // Fix up killed/dead flag after transformation.
4233 // ForwardKilledOperandReg = LI imm
4234 // y = XOP reg, ForwardKilledOperandReg(killed)
4235 if (ForwardKilledOperandReg != ~0U)
4236 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4240 const TargetRegisterClass *
4241 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
4242 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
4243 return &PPC::VSRCRegClass;
4247 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
4248 return PPC::getRecordFormOpcode(Opcode);
4251 // This function returns true if the machine instruction
4252 // always outputs a value by sign-extending a 32 bit value,
4253 // i.e. 0 to 31-th bits are same as 32-th bit.
4254 static bool isSignExtendingOp(const MachineInstr &MI) {
4255 int Opcode = MI.getOpcode();
4256 if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS ||
4257 Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec ||
4258 Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA ||
4259 Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 ||
4260 Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 ||
4261 Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX ||
4262 Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU ||
4263 Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
4264 Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 ||
4265 Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX ||
4266 Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB ||
4267 Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH ||
4268 Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 ||
4269 Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW ||
4270 Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 ||
4271 Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 ||
4272 Opcode == PPC::EXTSB8_32_64)
4275 if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33)
4278 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
4279 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
4280 MI.getOperand(3).getImm() > 0 &&
4281 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
4287 // This function returns true if the machine instruction
4288 // always outputs zeros in higher 32 bits.
4289 static bool isZeroExtendingOp(const MachineInstr &MI) {
4290 int Opcode = MI.getOpcode();
4291 // The 16-bit immediate is sign-extended in li/lis.
4292 // If the most significant bit is zero, all higher bits are zero.
4293 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
4294 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
4295 int64_t Imm = MI.getOperand(1).getImm();
4296 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
4300 // We have some variations of rotate-and-mask instructions
4301 // that clear higher 32-bits.
4302 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
4303 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
4304 Opcode == PPC::RLDICL_32_64) &&
4305 MI.getOperand(3).getImm() >= 32)
4308 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
4309 MI.getOperand(3).getImm() >= 32 &&
4310 MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm())
4313 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
4314 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
4315 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
4316 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm())
4319 // There are other instructions that clear higher 32-bits.
4320 if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec ||
4321 Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec ||
4322 Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 ||
4323 Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec ||
4324 Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec ||
4325 Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW ||
4326 Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec ||
4327 Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI ||
4328 Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI ||
4329 Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX ||
4330 Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX ||
4331 Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX ||
4332 Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ ||
4333 Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX ||
4334 Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 ||
4335 Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 ||
4336 Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 ||
4337 Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 ||
4338 Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 ||
4339 Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec ||
4340 Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec ||
4341 Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec ||
4342 Opcode == PPC::MFVSRWZ)
4348 // This function returns true if the input MachineInstr is a TOC save
4350 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
4351 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
4353 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
4354 unsigned StackOffset = MI.getOperand(1).getImm();
4355 Register StackReg = MI.getOperand(2).getReg();
4356 if (StackReg == PPC::X1 && StackOffset == TOCSaveOffset)
4362 // We limit the max depth to track incoming values of PHIs or binary ops
4363 // (e.g. AND) to avoid excessive cost.
4364 const unsigned MAX_DEPTH = 1;
4367 PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt,
4368 const unsigned Depth) const {
4369 const MachineFunction *MF = MI.getParent()->getParent();
4370 const MachineRegisterInfo *MRI = &MF->getRegInfo();
4372 // If we know this instruction returns sign- or zero-extended result,
4374 if (SignExt ? isSignExtendingOp(MI):
4375 isZeroExtendingOp(MI))
4378 switch (MI.getOpcode()) {
4380 Register SrcReg = MI.getOperand(1).getReg();
4382 // In both ELFv1 and v2 ABI, method parameters and the return value
4383 // are sign- or zero-extended.
4384 if (MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
4385 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
4386 // We check the ZExt/SExt flags for a method parameter.
4387 if (MI.getParent()->getBasicBlock() ==
4388 &MF->getFunction().getEntryBlock()) {
4389 Register VReg = MI.getOperand(0).getReg();
4390 if (MF->getRegInfo().isLiveIn(VReg))
4391 return SignExt ? FuncInfo->isLiveInSExt(VReg) :
4392 FuncInfo->isLiveInZExt(VReg);
4395 // For a method return value, we check the ZExt/SExt flags in attribute.
4396 // We assume the following code sequence for method call.
4397 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
4398 // BL8_NOP @func,...
4399 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
4400 // %5 = COPY %x3; G8RC:%5
4401 if (SrcReg == PPC::X3) {
4402 const MachineBasicBlock *MBB = MI.getParent();
4403 MachineBasicBlock::const_instr_iterator II =
4404 MachineBasicBlock::const_instr_iterator(&MI);
4405 if (II != MBB->instr_begin() &&
4406 (--II)->getOpcode() == PPC::ADJCALLSTACKUP) {
4407 const MachineInstr &CallMI = *(--II);
4408 if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) {
4409 const Function *CalleeFn =
4410 dyn_cast<Function>(CallMI.getOperand(0).getGlobal());
4413 const IntegerType *IntTy =
4414 dyn_cast<IntegerType>(CalleeFn->getReturnType());
4415 const AttributeSet &Attrs =
4416 CalleeFn->getAttributes().getRetAttributes();
4417 if (IntTy && IntTy->getBitWidth() <= 32)
4418 return Attrs.hasAttribute(SignExt ? Attribute::SExt :
4425 // If this is a copy from another register, we recursively check source.
4426 if (!Register::isVirtualRegister(SrcReg))
4428 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4430 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4436 case PPC::ANDIS_rec:
4441 case PPC::ANDI8_rec:
4442 case PPC::ANDIS8_rec:
4447 // logical operation with 16-bit immediate does not change the upper bits.
4448 // So, we track the operand register as we do for register copy.
4449 Register SrcReg = MI.getOperand(1).getReg();
4450 if (!Register::isVirtualRegister(SrcReg))
4452 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4454 return isSignOrZeroExtended(*SrcMI, SignExt, Depth);
4459 // If all incoming values are sign-/zero-extended,
4460 // the output of OR, ISEL or PHI is also sign-/zero-extended.
4465 if (Depth >= MAX_DEPTH)
4468 // The input registers for PHI are operand 1, 3, ...
4469 // The input registers for others are operand 1 and 2.
4470 unsigned E = 3, D = 1;
4471 if (MI.getOpcode() == PPC::PHI) {
4472 E = MI.getNumOperands();
4476 for (unsigned I = 1; I != E; I += D) {
4477 if (MI.getOperand(I).isReg()) {
4478 Register SrcReg = MI.getOperand(I).getReg();
4479 if (!Register::isVirtualRegister(SrcReg))
4481 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg);
4482 if (SrcMI == NULL || !isSignOrZeroExtended(*SrcMI, SignExt, Depth+1))
4491 // If at least one of the incoming values of an AND is zero extended
4492 // then the output is also zero-extended. If both of the incoming values
4493 // are sign-extended then the output is also sign extended.
4496 if (Depth >= MAX_DEPTH)
4499 assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg());
4501 Register SrcReg1 = MI.getOperand(1).getReg();
4502 Register SrcReg2 = MI.getOperand(2).getReg();
4504 if (!Register::isVirtualRegister(SrcReg1) ||
4505 !Register::isVirtualRegister(SrcReg2))
4508 const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1);
4509 const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2);
4510 if (!MISrc1 || !MISrc2)
4514 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) &&
4515 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4517 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) ||
4518 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1);
4527 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
4528 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
4532 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
4533 MachineInstr *Loop, *EndLoop, *LoopCount;
4534 MachineFunction *MF;
4535 const TargetInstrInfo *TII;
4539 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
4540 MachineInstr *LoopCount)
4541 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
4542 MF(Loop->getParent()->getParent()),
4543 TII(MF->getSubtarget().getInstrInfo()) {
4544 // Inspect the Loop instruction up-front, as it may be deleted when we call
4545 // createTripCountGreaterCondition.
4546 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
4547 TripCount = LoopCount->getOperand(1).getImm();
4552 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4553 // Only ignore the terminator.
4554 return MI == EndLoop;
4558 createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
4559 SmallVectorImpl<MachineOperand> &Cond) override {
4560 if (TripCount == -1) {
4561 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4562 // so we don't need to generate any thing here.
4563 Cond.push_back(MachineOperand::CreateImm(0));
4564 Cond.push_back(MachineOperand::CreateReg(
4565 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
4570 return TripCount > TC;
4573 void setPreheader(MachineBasicBlock *NewPreheader) override {
4574 // Do nothing. We want the LOOP setup instruction to stay in the *old*
4575 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
4578 void adjustTripCount(int TripCountAdjust) override {
4579 // If the loop trip count is a compile-time value, then just change the
4581 if (LoopCount->getOpcode() == PPC::LI8 ||
4582 LoopCount->getOpcode() == PPC::LI) {
4583 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
4584 LoopCount->getOperand(1).setImm(TripCount);
4588 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
4589 // so we don't need to generate any thing here.
4592 void disposed() override {
4593 Loop->eraseFromParent();
4594 // Ensure the loop setup instruction is deleted too.
4595 LoopCount->eraseFromParent();
4600 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
4601 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
4602 // We really "analyze" only hardware loops right now.
4603 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
4604 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
4605 if (Preheader == LoopBB)
4606 Preheader = *std::next(LoopBB->pred_begin());
4607 MachineFunction *MF = Preheader->getParent();
4609 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
4610 SmallPtrSet<MachineBasicBlock *, 8> Visited;
4611 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
4612 Register LoopCountReg = LoopInst->getOperand(0).getReg();
4613 MachineRegisterInfo &MRI = MF->getRegInfo();
4614 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
4615 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
4621 MachineInstr *PPCInstrInfo::findLoopInstr(
4622 MachineBasicBlock &PreHeader,
4623 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
4625 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
4627 // The loop set-up instruction should be in preheader
4628 for (auto &I : PreHeader.instrs())
4629 if (I.getOpcode() == LOOPi)
4634 // Return true if get the base operand, byte offset of an instruction and the
4635 // memory width. Width is the size of memory that is being loaded/stored.
4636 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
4637 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
4638 unsigned &Width, const TargetRegisterInfo *TRI) const {
4639 if (!LdSt.mayLoadOrStore())
4642 // Handle only loads/stores with base register followed by immediate offset.
4643 if (LdSt.getNumExplicitOperands() != 3)
4645 if (!LdSt.getOperand(1).isImm() || !LdSt.getOperand(2).isReg())
4648 if (!LdSt.hasOneMemOperand())
4651 Width = (*LdSt.memoperands_begin())->getSize();
4652 Offset = LdSt.getOperand(1).getImm();
4653 BaseReg = &LdSt.getOperand(2);
4657 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
4658 const MachineInstr &MIa, const MachineInstr &MIb) const {
4659 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
4660 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
4662 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
4663 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4666 // Retrieve the base register, offset from the base register and width. Width
4667 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
4668 // base registers are identical, and the offset of a lower memory access +
4669 // the width doesn't overlap the offset of a higher memory access,
4670 // then the memory accesses are different.
4671 const TargetRegisterInfo *TRI = &getRegisterInfo();
4672 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
4673 int64_t OffsetA = 0, OffsetB = 0;
4674 unsigned int WidthA = 0, WidthB = 0;
4675 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
4676 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
4677 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
4678 int LowOffset = std::min(OffsetA, OffsetB);
4679 int HighOffset = std::max(OffsetA, OffsetB);
4680 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4681 if (LowOffset + LowWidth <= HighOffset)