1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// SI Implementation of TargetInstrInfo.
12 //===----------------------------------------------------------------------===//
14 #include "SIInstrInfo.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/ADT/iterator_range.h"
28 #include "llvm/Analysis/AliasAnalysis.h"
29 #include "llvm/Analysis/MemoryLocation.h"
30 #include "llvm/Analysis/ValueTracking.h"
31 #include "llvm/CodeGen/MachineBasicBlock.h"
32 #include "llvm/CodeGen/MachineDominators.h"
33 #include "llvm/CodeGen/MachineFrameInfo.h"
34 #include "llvm/CodeGen/MachineFunction.h"
35 #include "llvm/CodeGen/MachineInstr.h"
36 #include "llvm/CodeGen/MachineInstrBuilder.h"
37 #include "llvm/CodeGen/MachineInstrBundle.h"
38 #include "llvm/CodeGen/MachineMemOperand.h"
39 #include "llvm/CodeGen/MachineOperand.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/RegisterScavenging.h"
42 #include "llvm/CodeGen/ScheduleDAG.h"
43 #include "llvm/CodeGen/SelectionDAGNodes.h"
44 #include "llvm/CodeGen/TargetOpcodes.h"
45 #include "llvm/CodeGen/TargetRegisterInfo.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/CommandLine.h"
54 #include "llvm/Support/Compiler.h"
55 #include "llvm/Support/ErrorHandling.h"
56 #include "llvm/Support/MachineValueType.h"
57 #include "llvm/Support/MathExtras.h"
58 #include "llvm/Target/TargetMachine.h"
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84 cl::desc("Restrict range of branch instructions (DEBUG)"));
86 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
87 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
92 //===----------------------------------------------------------------------===//
93 // TargetInstrInfo callbacks
94 //===----------------------------------------------------------------------===//
96 static unsigned getNumOperandsNoGlue(SDNode *Node) {
97 unsigned N = Node->getNumOperands();
98 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
103 /// Returns true if both nodes have the same value for the given
104 /// operand \p Op, or if both nodes do not have this operand.
105 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
106 unsigned Opc0 = N0->getMachineOpcode();
107 unsigned Opc1 = N1->getMachineOpcode();
109 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
110 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
112 if (Op0Idx == -1 && Op1Idx == -1)
116 if ((Op0Idx == -1 && Op1Idx != -1) ||
117 (Op1Idx == -1 && Op0Idx != -1))
120 // getNamedOperandIdx returns the index for the MachineInstr's operands,
121 // which includes the result as the first operand. We are indexing into the
122 // MachineSDNode's operands, so we need to skip the result operand to get
127 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
130 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
131 AliasAnalysis *AA) const {
132 // TODO: The generic check fails for VALU instructions that should be
133 // rematerializable due to implicit reads of exec. We really want all of the
134 // generic logic for this except for this.
135 switch (MI.getOpcode()) {
136 case AMDGPU::V_MOV_B32_e32:
137 case AMDGPU::V_MOV_B32_e64:
138 case AMDGPU::V_MOV_B64_PSEUDO:
139 // No implicit operands.
140 return MI.getNumOperands() == MI.getDesc().getNumOperands();
146 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
148 int64_t &Offset1) const {
149 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
152 unsigned Opc0 = Load0->getMachineOpcode();
153 unsigned Opc1 = Load1->getMachineOpcode();
155 // Make sure both are actually loads.
156 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
159 if (isDS(Opc0) && isDS(Opc1)) {
161 // FIXME: Handle this case:
162 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
166 if (Load0->getOperand(0) != Load1->getOperand(0))
169 // Skip read2 / write2 variants for simplicity.
170 // TODO: We should report true if the used offsets are adjacent (excluded
172 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
173 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
174 if (Offset0Idx == -1 || Offset1Idx == -1)
177 // XXX - be careful of datalesss loads
178 // getNamedOperandIdx returns the index for MachineInstrs. Since they
179 // include the output in the operand list, but SDNodes don't, we need to
180 // subtract the index by one.
181 Offset0Idx -= get(Opc0).NumDefs;
182 Offset1Idx -= get(Opc1).NumDefs;
183 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
184 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
188 if (isSMRD(Opc0) && isSMRD(Opc1)) {
189 // Skip time and cache invalidation instructions.
190 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
191 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
194 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1));
197 if (Load0->getOperand(0) != Load1->getOperand(0))
200 const ConstantSDNode *Load0Offset =
201 dyn_cast<ConstantSDNode>(Load0->getOperand(1));
202 const ConstantSDNode *Load1Offset =
203 dyn_cast<ConstantSDNode>(Load1->getOperand(1));
205 if (!Load0Offset || !Load1Offset)
208 Offset0 = Load0Offset->getZExtValue();
209 Offset1 = Load1Offset->getZExtValue();
213 // MUBUF and MTBUF can access the same addresses.
214 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
216 // MUBUF and MTBUF have vaddr at different indices.
217 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
218 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
219 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
222 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
223 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
225 if (OffIdx0 == -1 || OffIdx1 == -1)
228 // getNamedOperandIdx returns the index for MachineInstrs. Since they
229 // include the output in the operand list, but SDNodes don't, we need to
230 // subtract the index by one.
231 OffIdx0 -= get(Opc0).NumDefs;
232 OffIdx1 -= get(Opc1).NumDefs;
234 SDValue Off0 = Load0->getOperand(OffIdx0);
235 SDValue Off1 = Load1->getOperand(OffIdx1);
237 // The offset might be a FrameIndexSDNode.
238 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
241 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
242 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
249 static bool isStride64(unsigned Opc) {
251 case AMDGPU::DS_READ2ST64_B32:
252 case AMDGPU::DS_READ2ST64_B64:
253 case AMDGPU::DS_WRITE2ST64_B32:
254 case AMDGPU::DS_WRITE2ST64_B64:
261 bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt,
262 const MachineOperand *&BaseOp,
264 const TargetRegisterInfo *TRI) const {
265 if (!LdSt.mayLoadOrStore())
268 unsigned Opc = LdSt.getOpcode();
271 const MachineOperand *OffsetImm =
272 getNamedOperand(LdSt, AMDGPU::OpName::offset);
274 // Normal, single offset LDS instruction.
275 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
276 // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
278 if (!BaseOp || !BaseOp->isReg())
281 Offset = OffsetImm->getImm();
286 // The 2 offset instructions use offset0 and offset1 instead. We can treat
287 // these as a load with a single offset if the 2 offsets are consecutive. We
288 // will use this for some partially aligned loads.
289 const MachineOperand *Offset0Imm =
290 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
291 const MachineOperand *Offset1Imm =
292 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
294 uint8_t Offset0 = Offset0Imm->getImm();
295 uint8_t Offset1 = Offset1Imm->getImm();
297 if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
298 // Each of these offsets is in element sized units, so we need to convert
299 // to bytes of the individual reads.
303 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
305 assert(LdSt.mayStore());
306 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
307 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
313 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
314 if (!BaseOp->isReg())
317 Offset = EltSize * Offset0;
325 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
326 const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
327 if (SOffset && SOffset->isReg()) {
328 // We can only handle this if it's a stack access, as any other resource
329 // would require reporting multiple base registers.
330 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
331 if (AddrReg && !AddrReg->isFI())
334 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
335 const SIMachineFunctionInfo *MFI
336 = LdSt.getParent()->getParent()->getInfo<SIMachineFunctionInfo>();
337 if (RSrc->getReg() != MFI->getScratchRSrcReg())
340 const MachineOperand *OffsetImm =
341 getNamedOperand(LdSt, AMDGPU::OpName::offset);
343 Offset = OffsetImm->getImm();
347 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
351 const MachineOperand *OffsetImm =
352 getNamedOperand(LdSt, AMDGPU::OpName::offset);
354 Offset = OffsetImm->getImm();
355 if (SOffset) // soffset can be an inline immediate.
356 Offset += SOffset->getImm();
358 if (!BaseOp->isReg())
365 const MachineOperand *OffsetImm =
366 getNamedOperand(LdSt, AMDGPU::OpName::offset);
370 const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
372 Offset = OffsetImm->getImm();
373 if (!BaseOp->isReg())
380 const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
382 // Can't analyze 2 offsets.
383 if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
388 // scratch instructions have either vaddr or saddr.
389 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
392 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
393 if (!BaseOp->isReg())
401 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
402 const MachineOperand &BaseOp1,
403 const MachineInstr &MI2,
404 const MachineOperand &BaseOp2) {
405 // Support only base operands with base registers.
406 // Note: this could be extended to support FI operands.
407 if (!BaseOp1.isReg() || !BaseOp2.isReg())
410 if (BaseOp1.isIdenticalTo(BaseOp2))
413 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
416 auto MO1 = *MI1.memoperands_begin();
417 auto MO2 = *MI2.memoperands_begin();
418 if (MO1->getAddrSpace() != MO2->getAddrSpace())
421 auto Base1 = MO1->getValue();
422 auto Base2 = MO2->getValue();
423 if (!Base1 || !Base2)
425 const MachineFunction &MF = *MI1.getParent()->getParent();
426 const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
427 Base1 = GetUnderlyingObject(Base1, DL);
428 Base2 = GetUnderlyingObject(Base2, DL);
430 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
433 return Base1 == Base2;
436 bool SIInstrInfo::shouldClusterMemOps(const MachineOperand &BaseOp1,
437 const MachineOperand &BaseOp2,
438 unsigned NumLoads) const {
439 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
440 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
442 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
445 const MachineOperand *FirstDst = nullptr;
446 const MachineOperand *SecondDst = nullptr;
448 if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
449 (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
450 (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
451 const unsigned MaxGlobalLoadCluster = 6;
452 if (NumLoads > MaxGlobalLoadCluster)
455 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
457 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
458 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
460 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
461 } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
462 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
463 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
464 } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
465 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
466 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
469 if (!FirstDst || !SecondDst)
472 // Try to limit clustering based on the total number of bytes loaded
473 // rather than the number of instructions. This is done to help reduce
474 // register pressure. The method used is somewhat inexact, though,
475 // because it assumes that all loads in the cluster will load the
476 // same number of bytes as FirstLdSt.
478 // The unit of this value is bytes.
479 // FIXME: This needs finer tuning.
480 unsigned LoadClusterThreshold = 16;
482 const MachineRegisterInfo &MRI =
483 FirstLdSt.getParent()->getParent()->getRegInfo();
485 const Register Reg = FirstDst->getReg();
487 const TargetRegisterClass *DstRC = Register::isVirtualRegister(Reg)
488 ? MRI.getRegClass(Reg)
489 : RI.getPhysRegClass(Reg);
491 return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
494 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
495 // the first 16 loads will be interleaved with the stores, and the next 16 will
496 // be clustered as expected. It should really split into 2 16 store batches.
498 // Loads are clustered until this returns false, rather than trying to schedule
499 // groups of stores. This also means we have to deal with saying different
500 // address space loads should be clustered, and ones which might cause bank
503 // This might be deprecated so it might not be worth that much effort to fix.
504 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
505 int64_t Offset0, int64_t Offset1,
506 unsigned NumLoads) const {
507 assert(Offset1 > Offset0 &&
508 "Second offset should be larger than first offset!");
509 // If we have less than 16 loads in a row, and the offsets are within 64
510 // bytes, then schedule together.
512 // A cacheline is 64 bytes (for global memory).
513 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
516 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
517 MachineBasicBlock::iterator MI,
518 const DebugLoc &DL, MCRegister DestReg,
519 MCRegister SrcReg, bool KillSrc) {
520 MachineFunction *MF = MBB.getParent();
521 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
522 "illegal SGPR to VGPR copy",
524 LLVMContext &C = MF->getFunction().getContext();
525 C.diagnose(IllegalCopy);
527 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
528 .addReg(SrcReg, getKillRegState(KillSrc));
531 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
532 MachineBasicBlock::iterator MI,
533 const DebugLoc &DL, MCRegister DestReg,
534 MCRegister SrcReg, bool KillSrc) const {
535 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
537 if (RC == &AMDGPU::VGPR_32RegClass) {
538 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
539 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
540 AMDGPU::AGPR_32RegClass.contains(SrcReg));
541 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
542 AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32;
543 BuildMI(MBB, MI, DL, get(Opc), DestReg)
544 .addReg(SrcReg, getKillRegState(KillSrc));
548 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
549 RC == &AMDGPU::SReg_32RegClass) {
550 if (SrcReg == AMDGPU::SCC) {
551 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
557 if (DestReg == AMDGPU::VCC_LO) {
558 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
559 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
560 .addReg(SrcReg, getKillRegState(KillSrc));
562 // FIXME: Hack until VReg_1 removed.
563 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
564 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
566 .addReg(SrcReg, getKillRegState(KillSrc));
572 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
573 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
577 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
578 .addReg(SrcReg, getKillRegState(KillSrc));
582 if (RC == &AMDGPU::SReg_64RegClass) {
583 if (DestReg == AMDGPU::VCC) {
584 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
585 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
586 .addReg(SrcReg, getKillRegState(KillSrc));
588 // FIXME: Hack until VReg_1 removed.
589 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
590 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
592 .addReg(SrcReg, getKillRegState(KillSrc));
598 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
599 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
603 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
604 .addReg(SrcReg, getKillRegState(KillSrc));
608 if (DestReg == AMDGPU::SCC) {
609 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
610 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
611 .addReg(SrcReg, getKillRegState(KillSrc))
616 if (RC == &AMDGPU::AGPR_32RegClass) {
617 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
618 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
619 AMDGPU::AGPR_32RegClass.contains(SrcReg));
620 if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
621 // First try to find defining accvgpr_write to avoid temporary registers.
622 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
624 if (!Def->definesRegister(SrcReg, &RI))
626 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
629 MachineOperand &DefOp = Def->getOperand(1);
630 assert(DefOp.isReg() || DefOp.isImm());
633 // Check that register source operand if not clobbered before MI.
634 // Immediate operands are always safe to propagate.
635 bool SafeToPropagate = true;
636 for (auto I = Def; I != MI && SafeToPropagate; ++I)
637 if (I->modifiesRegister(DefOp.getReg(), &RI))
638 SafeToPropagate = false;
640 if (!SafeToPropagate)
643 DefOp.setIsKill(false);
646 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
652 RS.enterBasicBlock(MBB);
655 // Ideally we want to have three registers for a long reg_sequence copy
656 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
657 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
660 // Registers in the sequence are allocated contiguously so we can just
661 // use register number to pick one of three round-robin temps.
662 unsigned RegNo = DestReg % 3;
663 unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
665 report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
667 // Only loop through if there are any free registers left, otherwise
668 // scavenger may report a fatal error without emergency spill slot
669 // or spill with the slot.
670 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
671 unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
672 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
677 copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
678 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
679 .addReg(Tmp, RegState::Kill);
683 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
684 .addReg(SrcReg, getKillRegState(KillSrc));
688 unsigned EltSize = 4;
689 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
690 if (RI.isSGPRClass(RC)) {
691 // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
692 if (!(RI.getRegSizeInBits(*RC) % 64)) {
693 Opcode = AMDGPU::S_MOV_B64;
696 Opcode = AMDGPU::S_MOV_B32;
700 if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
701 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
704 } else if (RI.hasAGPRs(RC)) {
705 Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
706 AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
707 } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
708 Opcode = AMDGPU::V_ACCVGPR_READ_B32;
711 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
712 bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
714 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
717 SubIdx = SubIndices[Idx];
719 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
721 if (Opcode == TargetOpcode::COPY) {
722 copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
723 RI.getSubReg(SrcReg, SubIdx), KillSrc);
727 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
728 get(Opcode), RI.getSubReg(DestReg, SubIdx));
730 Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
733 Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
735 bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
736 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
740 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
743 // Try to map original to commuted opcode
744 NewOpc = AMDGPU::getCommuteRev(Opcode);
746 // Check if the commuted (REV) opcode exists on the target.
747 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
749 // Try to map commuted to original opcode
750 NewOpc = AMDGPU::getCommuteOrig(Opcode);
752 // Check if the original (non-REV) opcode exists on the target.
753 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
758 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
759 MachineBasicBlock::iterator MI,
760 const DebugLoc &DL, unsigned DestReg,
761 int64_t Value) const {
762 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
763 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
764 if (RegClass == &AMDGPU::SReg_32RegClass ||
765 RegClass == &AMDGPU::SGPR_32RegClass ||
766 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
767 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
768 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
773 if (RegClass == &AMDGPU::SReg_64RegClass ||
774 RegClass == &AMDGPU::SGPR_64RegClass ||
775 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
776 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
781 if (RegClass == &AMDGPU::VGPR_32RegClass) {
782 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
786 if (RegClass == &AMDGPU::VReg_64RegClass) {
787 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
792 unsigned EltSize = 4;
793 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
794 if (RI.isSGPRClass(RegClass)) {
795 if (RI.getRegSizeInBits(*RegClass) > 32) {
796 Opcode = AMDGPU::S_MOV_B64;
799 Opcode = AMDGPU::S_MOV_B32;
804 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
805 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
806 int64_t IdxValue = Idx == 0 ? Value : 0;
808 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
809 get(Opcode), RI.getSubReg(DestReg, Idx));
810 Builder.addImm(IdxValue);
814 const TargetRegisterClass *
815 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
816 return &AMDGPU::VGPR_32RegClass;
819 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
820 MachineBasicBlock::iterator I,
821 const DebugLoc &DL, unsigned DstReg,
822 ArrayRef<MachineOperand> Cond,
824 unsigned FalseReg) const {
825 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
826 MachineFunction *MF = MBB.getParent();
827 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
828 const TargetRegisterClass *BoolXExecRC =
829 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
830 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
833 if (Cond.size() == 1) {
834 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
835 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
837 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
843 } else if (Cond.size() == 2) {
844 assert(Cond[0].isImm() && "Cond[0] is not an immediate");
845 switch (Cond[0].getImm()) {
846 case SIInstrInfo::SCC_TRUE: {
847 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
848 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
849 : AMDGPU::S_CSELECT_B64), SReg)
852 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
860 case SIInstrInfo::SCC_FALSE: {
861 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
862 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
863 : AMDGPU::S_CSELECT_B64), SReg)
866 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
874 case SIInstrInfo::VCCNZ: {
875 MachineOperand RegOp = Cond[1];
876 RegOp.setImplicit(false);
877 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
878 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
880 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
888 case SIInstrInfo::VCCZ: {
889 MachineOperand RegOp = Cond[1];
890 RegOp.setImplicit(false);
891 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
892 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
894 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
902 case SIInstrInfo::EXECNZ: {
903 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
904 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
905 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
906 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
908 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
909 : AMDGPU::S_CSELECT_B64), SReg)
912 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
920 case SIInstrInfo::EXECZ: {
921 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
922 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
923 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
924 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
926 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
927 : AMDGPU::S_CSELECT_B64), SReg)
930 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
936 llvm_unreachable("Unhandled branch predicate EXECZ");
940 llvm_unreachable("invalid branch predicate");
943 llvm_unreachable("Can only handle Cond size 1 or 2");
947 unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
948 MachineBasicBlock::iterator I,
950 unsigned SrcReg, int Value) const {
951 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
952 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
953 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
960 unsigned SIInstrInfo::insertNE(MachineBasicBlock *MBB,
961 MachineBasicBlock::iterator I,
963 unsigned SrcReg, int Value) const {
964 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
965 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
966 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
973 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
975 if (RI.hasAGPRs(DstRC))
977 if (RI.getRegSizeInBits(*DstRC) == 32) {
978 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
979 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
980 return AMDGPU::S_MOV_B64;
981 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
982 return AMDGPU::V_MOV_B64_PSEUDO;
987 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
990 return AMDGPU::SI_SPILL_S32_SAVE;
992 return AMDGPU::SI_SPILL_S64_SAVE;
994 return AMDGPU::SI_SPILL_S96_SAVE;
996 return AMDGPU::SI_SPILL_S128_SAVE;
998 return AMDGPU::SI_SPILL_S160_SAVE;
1000 return AMDGPU::SI_SPILL_S256_SAVE;
1002 return AMDGPU::SI_SPILL_S512_SAVE;
1004 return AMDGPU::SI_SPILL_S1024_SAVE;
1006 llvm_unreachable("unknown register size");
1010 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1013 return AMDGPU::SI_SPILL_V32_SAVE;
1015 return AMDGPU::SI_SPILL_V64_SAVE;
1017 return AMDGPU::SI_SPILL_V96_SAVE;
1019 return AMDGPU::SI_SPILL_V128_SAVE;
1021 return AMDGPU::SI_SPILL_V160_SAVE;
1023 return AMDGPU::SI_SPILL_V256_SAVE;
1025 return AMDGPU::SI_SPILL_V512_SAVE;
1027 return AMDGPU::SI_SPILL_V1024_SAVE;
1029 llvm_unreachable("unknown register size");
1033 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1036 return AMDGPU::SI_SPILL_A32_SAVE;
1038 return AMDGPU::SI_SPILL_A64_SAVE;
1040 return AMDGPU::SI_SPILL_A128_SAVE;
1042 return AMDGPU::SI_SPILL_A512_SAVE;
1044 return AMDGPU::SI_SPILL_A1024_SAVE;
1046 llvm_unreachable("unknown register size");
1050 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1051 MachineBasicBlock::iterator MI,
1052 unsigned SrcReg, bool isKill,
1054 const TargetRegisterClass *RC,
1055 const TargetRegisterInfo *TRI) const {
1056 MachineFunction *MF = MBB.getParent();
1057 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1058 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1059 const DebugLoc &DL = MBB.findDebugLoc(MI);
1061 unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1062 unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1063 MachinePointerInfo PtrInfo
1064 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1065 MachineMemOperand *MMO
1066 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
1068 unsigned SpillSize = TRI->getSpillSize(*RC);
1070 if (RI.isSGPRClass(RC)) {
1071 MFI->setHasSpilledSGPRs();
1072 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1074 // We are only allowed to create one new instruction when spilling
1075 // registers, so we need to use pseudo instruction for spilling SGPRs.
1076 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1078 // The SGPR spill/restore instructions only work on number sgprs, so we need
1079 // to make sure we are using the correct register class.
1080 if (Register::isVirtualRegister(SrcReg) && SpillSize == 4) {
1081 MachineRegisterInfo &MRI = MF->getRegInfo();
1082 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1085 BuildMI(MBB, MI, DL, OpDesc)
1086 .addReg(SrcReg, getKillRegState(isKill)) // data
1087 .addFrameIndex(FrameIndex) // addr
1089 .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1090 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1091 // Add the scratch resource registers as implicit uses because we may end up
1092 // needing them, and need to ensure that the reserved registers are
1093 // correctly handled.
1094 if (RI.spillSGPRToVGPR())
1095 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1099 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1100 : getVGPRSpillSaveOpcode(SpillSize);
1101 MFI->setHasSpilledVGPRs();
1103 auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1104 if (RI.hasAGPRs(RC)) {
1105 MachineRegisterInfo &MRI = MF->getRegInfo();
1106 Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1107 MIB.addReg(Tmp, RegState::Define);
1109 MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1110 .addFrameIndex(FrameIndex) // addr
1111 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1112 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1113 .addImm(0) // offset
1114 .addMemOperand(MMO);
1117 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1120 return AMDGPU::SI_SPILL_S32_RESTORE;
1122 return AMDGPU::SI_SPILL_S64_RESTORE;
1124 return AMDGPU::SI_SPILL_S96_RESTORE;
1126 return AMDGPU::SI_SPILL_S128_RESTORE;
1128 return AMDGPU::SI_SPILL_S160_RESTORE;
1130 return AMDGPU::SI_SPILL_S256_RESTORE;
1132 return AMDGPU::SI_SPILL_S512_RESTORE;
1134 return AMDGPU::SI_SPILL_S1024_RESTORE;
1136 llvm_unreachable("unknown register size");
1140 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1143 return AMDGPU::SI_SPILL_V32_RESTORE;
1145 return AMDGPU::SI_SPILL_V64_RESTORE;
1147 return AMDGPU::SI_SPILL_V96_RESTORE;
1149 return AMDGPU::SI_SPILL_V128_RESTORE;
1151 return AMDGPU::SI_SPILL_V160_RESTORE;
1153 return AMDGPU::SI_SPILL_V256_RESTORE;
1155 return AMDGPU::SI_SPILL_V512_RESTORE;
1157 return AMDGPU::SI_SPILL_V1024_RESTORE;
1159 llvm_unreachable("unknown register size");
1163 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1166 return AMDGPU::SI_SPILL_A32_RESTORE;
1168 return AMDGPU::SI_SPILL_A64_RESTORE;
1170 return AMDGPU::SI_SPILL_A128_RESTORE;
1172 return AMDGPU::SI_SPILL_A512_RESTORE;
1174 return AMDGPU::SI_SPILL_A1024_RESTORE;
1176 llvm_unreachable("unknown register size");
1180 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1181 MachineBasicBlock::iterator MI,
1182 unsigned DestReg, int FrameIndex,
1183 const TargetRegisterClass *RC,
1184 const TargetRegisterInfo *TRI) const {
1185 MachineFunction *MF = MBB.getParent();
1186 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1187 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1188 const DebugLoc &DL = MBB.findDebugLoc(MI);
1189 unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1190 unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1191 unsigned SpillSize = TRI->getSpillSize(*RC);
1193 MachinePointerInfo PtrInfo
1194 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1196 MachineMemOperand *MMO = MF->getMachineMemOperand(
1197 PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1199 if (RI.isSGPRClass(RC)) {
1200 MFI->setHasSpilledSGPRs();
1201 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1203 // FIXME: Maybe this should not include a memoperand because it will be
1204 // lowered to non-memory instructions.
1205 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1206 if (Register::isVirtualRegister(DestReg) && SpillSize == 4) {
1207 MachineRegisterInfo &MRI = MF->getRegInfo();
1208 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1211 if (RI.spillSGPRToVGPR())
1212 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1213 BuildMI(MBB, MI, DL, OpDesc, DestReg)
1214 .addFrameIndex(FrameIndex) // addr
1216 .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1217 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1221 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1222 : getVGPRSpillRestoreOpcode(SpillSize);
1223 auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1224 if (RI.hasAGPRs(RC)) {
1225 MachineRegisterInfo &MRI = MF->getRegInfo();
1226 Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1227 MIB.addReg(Tmp, RegState::Define);
1229 MIB.addFrameIndex(FrameIndex) // vaddr
1230 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1231 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1232 .addImm(0) // offset
1233 .addMemOperand(MMO);
1236 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1237 unsigned SIInstrInfo::calculateLDSSpillAddress(
1238 MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1239 unsigned FrameOffset, unsigned Size) const {
1240 MachineFunction *MF = MBB.getParent();
1241 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1242 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1243 const DebugLoc &DL = MBB.findDebugLoc(MI);
1244 unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1245 unsigned WavefrontSize = ST.getWavefrontSize();
1247 unsigned TIDReg = MFI->getTIDReg();
1248 if (!MFI->hasCalculatedTID()) {
1249 MachineBasicBlock &Entry = MBB.getParent()->front();
1250 MachineBasicBlock::iterator Insert = Entry.front();
1251 const DebugLoc &DL = Insert->getDebugLoc();
1253 TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1255 if (TIDReg == AMDGPU::NoRegister)
1258 if (!AMDGPU::isShader(MF->getFunction().getCallingConv()) &&
1259 WorkGroupSize > WavefrontSize) {
1260 Register TIDIGXReg =
1261 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
1262 Register TIDIGYReg =
1263 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
1264 Register TIDIGZReg =
1265 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
1266 Register InputPtrReg =
1267 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1268 for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1269 if (!Entry.isLiveIn(Reg))
1270 Entry.addLiveIn(Reg);
1273 RS->enterBasicBlock(Entry);
1274 // FIXME: Can we scavenge an SReg_64 and access the subregs?
1275 unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1276 unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1277 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1278 .addReg(InputPtrReg)
1279 .addImm(SI::KernelInputOffsets::NGROUPS_Z);
1280 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1281 .addReg(InputPtrReg)
1282 .addImm(SI::KernelInputOffsets::NGROUPS_Y);
1284 // NGROUPS.X * NGROUPS.Y
1285 BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1288 // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1289 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1292 // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1293 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1297 // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1298 getAddNoCarry(Entry, Insert, DL, TIDReg)
1301 .addImm(0); // clamp bit
1304 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1309 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1315 BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1319 MFI->setTIDReg(TIDReg);
1322 // Add FrameIndex to LDS offset
1323 unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1324 getAddNoCarry(MBB, MI, DL, TmpReg)
1327 .addImm(0); // clamp bit
1332 void SIInstrInfo::insertWaitStates(MachineBasicBlock &MBB,
1333 MachineBasicBlock::iterator MI,
1335 DebugLoc DL = MBB.findDebugLoc(MI);
1343 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1348 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1349 MachineBasicBlock::iterator MI) const {
1350 insertWaitStates(MBB, MI, 1);
1353 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
1354 auto MF = MBB.getParent();
1355 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1357 assert(Info->isEntryFunction());
1359 if (MBB.succ_empty()) {
1360 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1361 if (HasNoTerminator) {
1362 if (Info->returnsVoid()) {
1363 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1365 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1371 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
1372 switch (MI.getOpcode()) {
1373 default: return 1; // FIXME: Do wait states equal cycles?
1376 return MI.getOperand(0).getImm() + 1;
1380 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1381 MachineBasicBlock &MBB = *MI.getParent();
1382 DebugLoc DL = MBB.findDebugLoc(MI);
1383 switch (MI.getOpcode()) {
1384 default: return TargetInstrInfo::expandPostRAPseudo(MI);
1385 case AMDGPU::S_MOV_B64_term:
1386 // This is only a terminator to get the correct spill code placement during
1387 // register allocation.
1388 MI.setDesc(get(AMDGPU::S_MOV_B64));
1391 case AMDGPU::S_MOV_B32_term:
1392 // This is only a terminator to get the correct spill code placement during
1393 // register allocation.
1394 MI.setDesc(get(AMDGPU::S_MOV_B32));
1397 case AMDGPU::S_XOR_B64_term:
1398 // This is only a terminator to get the correct spill code placement during
1399 // register allocation.
1400 MI.setDesc(get(AMDGPU::S_XOR_B64));
1403 case AMDGPU::S_XOR_B32_term:
1404 // This is only a terminator to get the correct spill code placement during
1405 // register allocation.
1406 MI.setDesc(get(AMDGPU::S_XOR_B32));
1409 case AMDGPU::S_OR_B32_term:
1410 // This is only a terminator to get the correct spill code placement during
1411 // register allocation.
1412 MI.setDesc(get(AMDGPU::S_OR_B32));
1415 case AMDGPU::S_ANDN2_B64_term:
1416 // This is only a terminator to get the correct spill code placement during
1417 // register allocation.
1418 MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1421 case AMDGPU::S_ANDN2_B32_term:
1422 // This is only a terminator to get the correct spill code placement during
1423 // register allocation.
1424 MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1427 case AMDGPU::V_MOV_B64_PSEUDO: {
1428 Register Dst = MI.getOperand(0).getReg();
1429 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1430 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1432 const MachineOperand &SrcOp = MI.getOperand(1);
1433 // FIXME: Will this work for 64-bit floating point immediates?
1434 assert(!SrcOp.isFPImm());
1435 if (SrcOp.isImm()) {
1436 APInt Imm(64, SrcOp.getImm());
1437 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1438 .addImm(Imm.getLoBits(32).getZExtValue())
1439 .addReg(Dst, RegState::Implicit | RegState::Define);
1440 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1441 .addImm(Imm.getHiBits(32).getZExtValue())
1442 .addReg(Dst, RegState::Implicit | RegState::Define);
1444 assert(SrcOp.isReg());
1445 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1446 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1447 .addReg(Dst, RegState::Implicit | RegState::Define);
1448 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1449 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1450 .addReg(Dst, RegState::Implicit | RegState::Define);
1452 MI.eraseFromParent();
1455 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
1459 case AMDGPU::V_SET_INACTIVE_B32: {
1460 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1461 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1462 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1464 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1465 .add(MI.getOperand(2));
1466 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1468 MI.eraseFromParent();
1471 case AMDGPU::V_SET_INACTIVE_B64: {
1472 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1473 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1474 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1476 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1477 MI.getOperand(0).getReg())
1478 .add(MI.getOperand(2));
1479 expandPostRAPseudo(*Copy);
1480 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1482 MI.eraseFromParent();
1485 case AMDGPU::V_MOVRELD_B32_V1:
1486 case AMDGPU::V_MOVRELD_B32_V2:
1487 case AMDGPU::V_MOVRELD_B32_V4:
1488 case AMDGPU::V_MOVRELD_B32_V8:
1489 case AMDGPU::V_MOVRELD_B32_V16: {
1490 const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1491 Register VecReg = MI.getOperand(0).getReg();
1492 bool IsUndef = MI.getOperand(1).isUndef();
1493 unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1494 assert(VecReg == MI.getOperand(1).getReg());
1496 MachineInstr *MovRel =
1497 BuildMI(MBB, MI, DL, MovRelDesc)
1498 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1499 .add(MI.getOperand(2))
1500 .addReg(VecReg, RegState::ImplicitDefine)
1502 RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1504 const int ImpDefIdx =
1505 MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1506 const int ImpUseIdx = ImpDefIdx + 1;
1507 MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1509 MI.eraseFromParent();
1512 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1513 MachineFunction &MF = *MBB.getParent();
1514 Register Reg = MI.getOperand(0).getReg();
1515 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1516 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1518 // Create a bundle so these instructions won't be re-ordered by the
1519 // post-RA scheduler.
1520 MIBundleBuilder Bundler(MBB, MI);
1521 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1523 // Add 32-bit offset from this instruction to the start of the
1525 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1527 .add(MI.getOperand(1)));
1529 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1531 MIB.add(MI.getOperand(2));
1533 Bundler.append(MIB);
1534 finalizeBundle(MBB, Bundler.begin());
1536 MI.eraseFromParent();
1539 case AMDGPU::ENTER_WWM: {
1540 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1542 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1543 : AMDGPU::S_OR_SAVEEXEC_B64));
1546 case AMDGPU::EXIT_WWM: {
1547 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1549 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1552 case TargetOpcode::BUNDLE: {
1553 if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1556 // If it is a load it must be a memory clause
1557 for (MachineBasicBlock::instr_iterator I = MI.getIterator();
1558 I->isBundledWithSucc(); ++I) {
1559 I->unbundleFromSucc();
1560 for (MachineOperand &MO : I->operands())
1562 MO.setIsInternalRead(false);
1565 MI.eraseFromParent();
1572 std::pair<MachineInstr*, MachineInstr*>
1573 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
1574 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
1576 MachineBasicBlock &MBB = *MI.getParent();
1577 DebugLoc DL = MBB.findDebugLoc(MI);
1578 MachineFunction *MF = MBB.getParent();
1579 MachineRegisterInfo &MRI = MF->getRegInfo();
1580 Register Dst = MI.getOperand(0).getReg();
1582 MachineInstr *Split[2];
1585 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
1586 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
1587 if (Dst.isPhysical()) {
1588 MovDPP.addDef(RI.getSubReg(Dst, Sub));
1590 assert(MRI.isSSA());
1591 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1595 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
1596 const MachineOperand &SrcOp = MI.getOperand(I);
1597 assert(!SrcOp.isFPImm());
1598 if (SrcOp.isImm()) {
1599 APInt Imm(64, SrcOp.getImm());
1600 Imm.ashrInPlace(Part * 32);
1601 MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
1603 assert(SrcOp.isReg());
1604 Register Src = SrcOp.getReg();
1605 if (Src.isPhysical())
1606 MovDPP.addReg(RI.getSubReg(Src, Sub));
1608 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);
1612 for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I)
1613 MovDPP.addImm(MI.getOperand(I).getImm());
1615 Split[Part] = MovDPP;
1619 if (Dst.isVirtual())
1620 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
1621 .addReg(Split[0]->getOperand(0).getReg())
1622 .addImm(AMDGPU::sub0)
1623 .addReg(Split[1]->getOperand(0).getReg())
1624 .addImm(AMDGPU::sub1);
1626 MI.eraseFromParent();
1627 return std::make_pair(Split[0], Split[1]);
1630 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
1631 MachineOperand &Src0,
1632 unsigned Src0OpName,
1633 MachineOperand &Src1,
1634 unsigned Src1OpName) const {
1635 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1639 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1641 "All commutable instructions have both src0 and src1 modifiers");
1643 int Src0ModsVal = Src0Mods->getImm();
1644 int Src1ModsVal = Src1Mods->getImm();
1646 Src1Mods->setImm(Src0ModsVal);
1647 Src0Mods->setImm(Src1ModsVal);
1651 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
1652 MachineOperand &RegOp,
1653 MachineOperand &NonRegOp) {
1654 Register Reg = RegOp.getReg();
1655 unsigned SubReg = RegOp.getSubReg();
1656 bool IsKill = RegOp.isKill();
1657 bool IsDead = RegOp.isDead();
1658 bool IsUndef = RegOp.isUndef();
1659 bool IsDebug = RegOp.isDebug();
1661 if (NonRegOp.isImm())
1662 RegOp.ChangeToImmediate(NonRegOp.getImm());
1663 else if (NonRegOp.isFI())
1664 RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1668 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1669 NonRegOp.setSubReg(SubReg);
1674 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1676 unsigned Src1Idx) const {
1677 assert(!NewMI && "this should never be used");
1679 unsigned Opc = MI.getOpcode();
1680 int CommutedOpcode = commuteOpcode(Opc);
1681 if (CommutedOpcode == -1)
1684 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1685 static_cast<int>(Src0Idx) &&
1686 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1687 static_cast<int>(Src1Idx) &&
1688 "inconsistency with findCommutedOpIndices");
1690 MachineOperand &Src0 = MI.getOperand(Src0Idx);
1691 MachineOperand &Src1 = MI.getOperand(Src1Idx);
1693 MachineInstr *CommutedMI = nullptr;
1694 if (Src0.isReg() && Src1.isReg()) {
1695 if (isOperandLegal(MI, Src1Idx, &Src0)) {
1696 // Be sure to copy the source modifiers to the right place.
1698 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1701 } else if (Src0.isReg() && !Src1.isReg()) {
1702 // src0 should always be able to support any operand type, so no need to
1703 // check operand legality.
1704 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1705 } else if (!Src0.isReg() && Src1.isReg()) {
1706 if (isOperandLegal(MI, Src1Idx, &Src0))
1707 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1709 // FIXME: Found two non registers to commute. This does happen.
1714 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1715 Src1, AMDGPU::OpName::src1_modifiers);
1717 CommutedMI->setDesc(get(CommutedOpcode));
1723 // This needs to be implemented because the source modifiers may be inserted
1724 // between the true commutable operands, and the base
1725 // TargetInstrInfo::commuteInstruction uses it.
1726 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1727 unsigned &SrcOpIdx0,
1728 unsigned &SrcOpIdx1) const {
1729 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1732 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1733 unsigned &SrcOpIdx1) const {
1734 if (!Desc.isCommutable())
1737 unsigned Opc = Desc.getOpcode();
1738 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1742 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1746 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1749 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1750 int64_t BrOffset) const {
1751 // BranchRelaxation should never have to check s_setpc_b64 because its dest
1752 // block is unanalyzable.
1753 assert(BranchOp != AMDGPU::S_SETPC_B64);
1755 // Convert to dwords.
1758 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1759 // from the next instruction.
1762 return isIntN(BranchOffsetBits, BrOffset);
1765 MachineBasicBlock *SIInstrInfo::getBranchDestBlock(
1766 const MachineInstr &MI) const {
1767 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1768 // This would be a difficult analysis to perform, but can always be legal so
1769 // there's no need to analyze it.
1773 return MI.getOperand(0).getMBB();
1776 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1777 MachineBasicBlock &DestBB,
1780 RegScavenger *RS) const {
1781 assert(RS && "RegScavenger required for long branching");
1782 assert(MBB.empty() &&
1783 "new block should be inserted for expanding unconditional branch");
1784 assert(MBB.pred_size() == 1);
1786 MachineFunction *MF = MBB.getParent();
1787 MachineRegisterInfo &MRI = MF->getRegInfo();
1789 // FIXME: Virtual register workaround for RegScavenger not working with empty
1791 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1795 // We need to compute the offset relative to the instruction immediately after
1796 // s_getpc_b64. Insert pc arithmetic code before last terminator.
1797 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1799 // TODO: Handle > 32-bit block address.
1800 if (BrOffset >= 0) {
1801 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1802 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1803 .addReg(PCReg, 0, AMDGPU::sub0)
1804 .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1805 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1806 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1807 .addReg(PCReg, 0, AMDGPU::sub1)
1810 // Backwards branch.
1811 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1812 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1813 .addReg(PCReg, 0, AMDGPU::sub0)
1814 .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1815 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1816 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1817 .addReg(PCReg, 0, AMDGPU::sub1)
1821 // Insert the indirect branch after the other terminator.
1822 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1825 // FIXME: If spilling is necessary, this will fail because this scavenger has
1826 // no emergency stack slots. It is non-trivial to spill in this situation,
1827 // because the restore code needs to be specially placed after the
1828 // jump. BranchRelaxation then needs to be made aware of the newly inserted
1831 // If a spill is needed for the pc register pair, we need to insert a spill
1832 // restore block right before the destination block, and insert a short branch
1833 // into the old destination block's fallthrough predecessor.
1836 // s_cbranch_scc0 skip_long_branch:
1840 // s_getpc_b64 s[8:9]
1841 // s_add_u32 s8, s8, restore_bb
1842 // s_addc_u32 s9, s9, 0
1843 // s_setpc_b64 s[8:9]
1845 // skip_long_branch:
1850 // dest_bb_fallthrough_predecessor:
1856 // fallthrough dest_bb
1861 RS->enterBasicBlockEnd(MBB);
1862 unsigned Scav = RS->scavengeRegisterBackwards(
1863 AMDGPU::SReg_64RegClass,
1864 MachineBasicBlock::iterator(GetPC), false, 0);
1865 MRI.replaceRegWith(PCReg, Scav);
1866 MRI.clearVirtRegs();
1867 RS->setRegUsed(Scav);
1869 return 4 + 8 + 4 + 4;
1872 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1874 case SIInstrInfo::SCC_TRUE:
1875 return AMDGPU::S_CBRANCH_SCC1;
1876 case SIInstrInfo::SCC_FALSE:
1877 return AMDGPU::S_CBRANCH_SCC0;
1878 case SIInstrInfo::VCCNZ:
1879 return AMDGPU::S_CBRANCH_VCCNZ;
1880 case SIInstrInfo::VCCZ:
1881 return AMDGPU::S_CBRANCH_VCCZ;
1882 case SIInstrInfo::EXECNZ:
1883 return AMDGPU::S_CBRANCH_EXECNZ;
1884 case SIInstrInfo::EXECZ:
1885 return AMDGPU::S_CBRANCH_EXECZ;
1887 llvm_unreachable("invalid branch predicate");
1891 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1893 case AMDGPU::S_CBRANCH_SCC0:
1895 case AMDGPU::S_CBRANCH_SCC1:
1897 case AMDGPU::S_CBRANCH_VCCNZ:
1899 case AMDGPU::S_CBRANCH_VCCZ:
1901 case AMDGPU::S_CBRANCH_EXECNZ:
1903 case AMDGPU::S_CBRANCH_EXECZ:
1910 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
1911 MachineBasicBlock::iterator I,
1912 MachineBasicBlock *&TBB,
1913 MachineBasicBlock *&FBB,
1914 SmallVectorImpl<MachineOperand> &Cond,
1915 bool AllowModify) const {
1916 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1917 // Unconditional Branch
1918 TBB = I->getOperand(0).getMBB();
1922 MachineBasicBlock *CondBB = nullptr;
1924 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1925 CondBB = I->getOperand(1).getMBB();
1926 Cond.push_back(I->getOperand(0));
1928 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1929 if (Pred == INVALID_BR)
1932 CondBB = I->getOperand(0).getMBB();
1933 Cond.push_back(MachineOperand::CreateImm(Pred));
1934 Cond.push_back(I->getOperand(1)); // Save the branch register.
1938 if (I == MBB.end()) {
1939 // Conditional branch followed by fall-through.
1944 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1946 FBB = I->getOperand(0).getMBB();
1953 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
1954 MachineBasicBlock *&FBB,
1955 SmallVectorImpl<MachineOperand> &Cond,
1956 bool AllowModify) const {
1957 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1962 // Skip over the instructions that are artificially terminators for special
1964 while (I != E && !I->isBranch() && !I->isReturn() &&
1965 I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1966 switch (I->getOpcode()) {
1967 case AMDGPU::SI_MASK_BRANCH:
1968 case AMDGPU::S_MOV_B64_term:
1969 case AMDGPU::S_XOR_B64_term:
1970 case AMDGPU::S_ANDN2_B64_term:
1971 case AMDGPU::S_MOV_B32_term:
1972 case AMDGPU::S_XOR_B32_term:
1973 case AMDGPU::S_OR_B32_term:
1974 case AMDGPU::S_ANDN2_B32_term:
1977 case AMDGPU::SI_ELSE:
1978 case AMDGPU::SI_KILL_I1_TERMINATOR:
1979 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1980 // FIXME: It's messy that these need to be considered here at all.
1983 llvm_unreachable("unexpected non-branch terminator inst");
1992 if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1993 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1997 // TODO: Should be able to treat as fallthrough?
2001 if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
2004 MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
2006 // Specifically handle the case where the conditional branch is to the same
2007 // destination as the mask branch. e.g.
2009 // si_mask_branch BB8
2010 // s_cbranch_execz BB8
2013 // This is required to understand divergent loops which may need the branches
2015 if (TBB != MaskBrDest || Cond.empty())
2018 auto Pred = Cond[0].getImm();
2019 return (Pred != EXECZ && Pred != EXECNZ);
2022 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
2023 int *BytesRemoved) const {
2024 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
2027 unsigned RemovedSize = 0;
2028 while (I != MBB.end()) {
2029 MachineBasicBlock::iterator Next = std::next(I);
2030 if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
2035 RemovedSize += getInstSizeInBytes(*I);
2036 I->eraseFromParent();
2042 *BytesRemoved = RemovedSize;
2047 // Copy the flags onto the implicit condition register operand.
2048 static void preserveCondRegFlags(MachineOperand &CondReg,
2049 const MachineOperand &OrigCond) {
2050 CondReg.setIsUndef(OrigCond.isUndef());
2051 CondReg.setIsKill(OrigCond.isKill());
2054 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
2055 MachineBasicBlock *TBB,
2056 MachineBasicBlock *FBB,
2057 ArrayRef<MachineOperand> Cond,
2059 int *BytesAdded) const {
2060 if (!FBB && Cond.empty()) {
2061 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2068 if(Cond.size() == 1 && Cond[0].isReg()) {
2069 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2075 assert(TBB && Cond[0].isImm());
2078 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2082 MachineInstr *CondBr =
2083 BuildMI(&MBB, DL, get(Opcode))
2086 // Copy the flags onto the implicit condition register operand.
2087 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2096 MachineInstr *CondBr =
2097 BuildMI(&MBB, DL, get(Opcode))
2099 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2102 MachineOperand &CondReg = CondBr->getOperand(1);
2103 CondReg.setIsUndef(Cond[1].isUndef());
2104 CondReg.setIsKill(Cond[1].isKill());
2112 bool SIInstrInfo::reverseBranchCondition(
2113 SmallVectorImpl<MachineOperand> &Cond) const {
2114 if (Cond.size() != 2) {
2118 if (Cond[0].isImm()) {
2119 Cond[0].setImm(-Cond[0].getImm());
2126 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
2127 ArrayRef<MachineOperand> Cond,
2128 unsigned TrueReg, unsigned FalseReg,
2130 int &TrueCycles, int &FalseCycles) const {
2131 switch (Cond[0].getImm()) {
2134 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2135 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2136 assert(MRI.getRegClass(FalseReg) == RC);
2138 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2139 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2141 // Limit to equal cost for branch vs. N v_cndmask_b32s.
2142 return RI.hasVGPRs(RC) && NumInsts <= 6;
2146 // FIXME: We could insert for VGPRs if we could replace the original compare
2147 // with a vector one.
2148 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2149 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2150 assert(MRI.getRegClass(FalseReg) == RC);
2152 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2154 // Multiples of 8 can do s_cselect_b64
2155 if (NumInsts % 2 == 0)
2158 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2159 return RI.isSGPRClass(RC);
2166 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
2167 MachineBasicBlock::iterator I, const DebugLoc &DL,
2168 unsigned DstReg, ArrayRef<MachineOperand> Cond,
2169 unsigned TrueReg, unsigned FalseReg) const {
2170 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2171 if (Pred == VCCZ || Pred == SCC_FALSE) {
2172 Pred = static_cast<BranchPredicate>(-Pred);
2173 std::swap(TrueReg, FalseReg);
2176 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2177 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2178 unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2180 if (DstSize == 32) {
2181 unsigned SelOp = Pred == SCC_TRUE ?
2182 AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
2184 // Instruction's operands are backwards from what is expected.
2185 MachineInstr *Select =
2186 BuildMI(MBB, I, DL, get(SelOp), DstReg)
2190 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2194 if (DstSize == 64 && Pred == SCC_TRUE) {
2195 MachineInstr *Select =
2196 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2200 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2204 static const int16_t Sub0_15[] = {
2205 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2206 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2207 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2208 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2211 static const int16_t Sub0_15_64[] = {
2212 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2213 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2214 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2215 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2218 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2219 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2220 const int16_t *SubIndices = Sub0_15;
2221 int NElts = DstSize / 32;
2223 // 64-bit select is only available for SALU.
2224 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2225 if (Pred == SCC_TRUE) {
2227 SelOp = AMDGPU::S_CSELECT_B32;
2228 EltRC = &AMDGPU::SGPR_32RegClass;
2230 SelOp = AMDGPU::S_CSELECT_B64;
2231 EltRC = &AMDGPU::SGPR_64RegClass;
2232 SubIndices = Sub0_15_64;
2237 MachineInstrBuilder MIB = BuildMI(
2238 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2240 I = MIB->getIterator();
2242 SmallVector<unsigned, 8> Regs;
2243 for (int Idx = 0; Idx != NElts; ++Idx) {
2244 Register DstElt = MRI.createVirtualRegister(EltRC);
2245 Regs.push_back(DstElt);
2247 unsigned SubIdx = SubIndices[Idx];
2249 MachineInstr *Select =
2250 BuildMI(MBB, I, DL, get(SelOp), DstElt)
2251 .addReg(FalseReg, 0, SubIdx)
2252 .addReg(TrueReg, 0, SubIdx);
2253 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2254 fixImplicitOperands(*Select);
2261 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const {
2262 switch (MI.getOpcode()) {
2263 case AMDGPU::V_MOV_B32_e32:
2264 case AMDGPU::V_MOV_B32_e64:
2265 case AMDGPU::V_MOV_B64_PSEUDO: {
2266 // If there are additional implicit register operands, this may be used for
2267 // register indexing so the source register operand isn't simply copied.
2268 unsigned NumOps = MI.getDesc().getNumOperands() +
2269 MI.getDesc().getNumImplicitUses();
2271 return MI.getNumOperands() == NumOps;
2273 case AMDGPU::S_MOV_B32:
2274 case AMDGPU::S_MOV_B64:
2276 case AMDGPU::V_ACCVGPR_WRITE_B32:
2277 case AMDGPU::V_ACCVGPR_READ_B32:
2284 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind(
2285 unsigned Kind) const {
2287 case PseudoSourceValue::Stack:
2288 case PseudoSourceValue::FixedStack:
2289 return AMDGPUAS::PRIVATE_ADDRESS;
2290 case PseudoSourceValue::ConstantPool:
2291 case PseudoSourceValue::GOT:
2292 case PseudoSourceValue::JumpTable:
2293 case PseudoSourceValue::GlobalValueCallEntry:
2294 case PseudoSourceValue::ExternalSymbolCallEntry:
2295 case PseudoSourceValue::TargetCustom:
2296 return AMDGPUAS::CONSTANT_ADDRESS;
2298 return AMDGPUAS::FLAT_ADDRESS;
2301 static void removeModOperands(MachineInstr &MI) {
2302 unsigned Opc = MI.getOpcode();
2303 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2304 AMDGPU::OpName::src0_modifiers);
2305 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2306 AMDGPU::OpName::src1_modifiers);
2307 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2308 AMDGPU::OpName::src2_modifiers);
2310 MI.RemoveOperand(Src2ModIdx);
2311 MI.RemoveOperand(Src1ModIdx);
2312 MI.RemoveOperand(Src0ModIdx);
2315 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2316 unsigned Reg, MachineRegisterInfo *MRI) const {
2317 if (!MRI->hasOneNonDBGUse(Reg))
2320 switch (DefMI.getOpcode()) {
2323 case AMDGPU::S_MOV_B64:
2324 // TODO: We could fold 64-bit immediates, but this get compilicated
2325 // when there are sub-registers.
2328 case AMDGPU::V_MOV_B32_e32:
2329 case AMDGPU::S_MOV_B32:
2330 case AMDGPU::V_ACCVGPR_WRITE_B32:
2334 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2336 // FIXME: We could handle FrameIndex values here.
2337 if (!ImmOp->isImm())
2340 unsigned Opc = UseMI.getOpcode();
2341 if (Opc == AMDGPU::COPY) {
2342 bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2343 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2344 if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2345 if (!isInlineConstant(*ImmOp, AMDGPU::OPERAND_REG_INLINE_AC_INT32))
2347 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2349 UseMI.setDesc(get(NewOpc));
2350 UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2351 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2355 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2356 Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2357 Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2358 Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2359 // Don't fold if we are using source or output modifiers. The new VOP2
2360 // instructions don't have them.
2361 if (hasAnyModifiersSet(UseMI))
2364 // If this is a free constant, there's no reason to do this.
2365 // TODO: We could fold this here instead of letting SIFoldOperands do it
2367 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2369 // Any src operand can be used for the legality check.
2370 if (isInlineConstant(UseMI, *Src0, *ImmOp))
2373 bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2374 Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2375 bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2376 Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2377 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2378 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2380 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2381 // We should only expect these to be on src0 due to canonicalizations.
2382 if (Src0->isReg() && Src0->getReg() == Reg) {
2383 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2386 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2390 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2391 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2392 if (pseudoToMCOpcode(NewOpc) == -1)
2395 // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2397 const int64_t Imm = ImmOp->getImm();
2399 // FIXME: This would be a lot easier if we could return a new instruction
2400 // instead of having to modify in place.
2402 // Remove these first since they are at the end.
2403 UseMI.RemoveOperand(
2404 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2405 UseMI.RemoveOperand(
2406 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2408 Register Src1Reg = Src1->getReg();
2409 unsigned Src1SubReg = Src1->getSubReg();
2410 Src0->setReg(Src1Reg);
2411 Src0->setSubReg(Src1SubReg);
2412 Src0->setIsKill(Src1->isKill());
2414 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2415 Opc == AMDGPU::V_MAC_F16_e64 ||
2416 Opc == AMDGPU::V_FMAC_F32_e64 ||
2417 Opc == AMDGPU::V_FMAC_F16_e64)
2418 UseMI.untieRegOperand(
2419 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2421 Src1->ChangeToImmediate(Imm);
2423 removeModOperands(UseMI);
2424 UseMI.setDesc(get(NewOpc));
2426 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2428 DefMI.eraseFromParent();
2433 // Added part is the constant: Use v_madak_{f16, f32}.
2434 if (Src2->isReg() && Src2->getReg() == Reg) {
2435 // Not allowed to use constant bus for another operand.
2436 // We can however allow an inline immediate as src0.
2437 bool Src0Inlined = false;
2438 if (Src0->isReg()) {
2439 // Try to inline constant if possible.
2440 // If the Def moves immediate and the use is single
2441 // We are saving VGPR here.
2442 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2443 if (Def && Def->isMoveImmediate() &&
2444 isInlineConstant(Def->getOperand(1)) &&
2445 MRI->hasOneUse(Src0->getReg())) {
2446 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2448 } else if ((Register::isPhysicalRegister(Src0->getReg()) &&
2449 (ST.getConstantBusLimit(Opc) <= 1 &&
2450 RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2451 (Register::isVirtualRegister(Src0->getReg()) &&
2452 (ST.getConstantBusLimit(Opc) <= 1 &&
2453 RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2455 // VGPR is okay as Src0 - fallthrough
2458 if (Src1->isReg() && !Src0Inlined ) {
2459 // We have one slot for inlinable constant so far - try to fill it
2460 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2461 if (Def && Def->isMoveImmediate() &&
2462 isInlineConstant(Def->getOperand(1)) &&
2463 MRI->hasOneUse(Src1->getReg()) &&
2464 commuteInstruction(UseMI)) {
2465 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2466 } else if ((Register::isPhysicalRegister(Src1->getReg()) &&
2467 RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2468 (Register::isVirtualRegister(Src1->getReg()) &&
2469 RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2471 // VGPR is okay as Src1 - fallthrough
2475 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2476 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2477 if (pseudoToMCOpcode(NewOpc) == -1)
2480 const int64_t Imm = ImmOp->getImm();
2482 // FIXME: This would be a lot easier if we could return a new instruction
2483 // instead of having to modify in place.
2485 // Remove these first since they are at the end.
2486 UseMI.RemoveOperand(
2487 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2488 UseMI.RemoveOperand(
2489 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2491 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2492 Opc == AMDGPU::V_MAC_F16_e64 ||
2493 Opc == AMDGPU::V_FMAC_F32_e64 ||
2494 Opc == AMDGPU::V_FMAC_F16_e64)
2495 UseMI.untieRegOperand(
2496 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2498 // ChangingToImmediate adds Src2 back to the instruction.
2499 Src2->ChangeToImmediate(Imm);
2501 // These come before src2.
2502 removeModOperands(UseMI);
2503 UseMI.setDesc(get(NewOpc));
2504 // It might happen that UseMI was commuted
2505 // and we now have SGPR as SRC1. If so 2 inlined
2506 // constant and SGPR are illegal.
2507 legalizeOperands(UseMI);
2509 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2511 DefMI.eraseFromParent();
2520 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2521 int WidthB, int OffsetB) {
2522 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2523 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2524 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2525 return LowOffset + LowWidth <= HighOffset;
2528 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2529 const MachineInstr &MIb) const {
2530 const MachineOperand *BaseOp0, *BaseOp1;
2531 int64_t Offset0, Offset1;
2533 if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2534 getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2535 if (!BaseOp0->isIdenticalTo(*BaseOp1))
2538 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2539 // FIXME: Handle ds_read2 / ds_write2.
2542 unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2543 unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2544 if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2552 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
2553 const MachineInstr &MIb) const {
2554 assert(MIa.mayLoadOrStore() &&
2555 "MIa must load from or modify a memory location");
2556 assert(MIb.mayLoadOrStore() &&
2557 "MIb must load from or modify a memory location");
2559 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
2562 // XXX - Can we relax this between address spaces?
2563 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2566 // TODO: Should we check the address space from the MachineMemOperand? That
2567 // would allow us to distinguish objects we know don't alias based on the
2568 // underlying address space, even if it was lowered to a different one,
2569 // e.g. private accesses lowered to use MUBUF instructions on a scratch
2573 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2575 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2578 if (isMUBUF(MIa) || isMTBUF(MIa)) {
2579 if (isMUBUF(MIb) || isMTBUF(MIb))
2580 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2582 return !isFLAT(MIb) && !isSMRD(MIb);
2587 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2589 return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2594 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2602 static int64_t getFoldableImm(const MachineOperand* MO) {
2605 const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2606 const MachineRegisterInfo &MRI = MF->getRegInfo();
2607 auto Def = MRI.getUniqueVRegDef(MO->getReg());
2608 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2609 Def->getOperand(1).isImm())
2610 return Def->getOperand(1).getImm();
2611 return AMDGPU::NoRegister;
2614 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
2616 LiveVariables *LV) const {
2617 unsigned Opc = MI.getOpcode();
2619 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2620 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2625 case AMDGPU::V_MAC_F16_e64:
2626 case AMDGPU::V_FMAC_F16_e64:
2629 case AMDGPU::V_MAC_F32_e64:
2630 case AMDGPU::V_FMAC_F32_e64:
2632 case AMDGPU::V_MAC_F16_e32:
2633 case AMDGPU::V_FMAC_F16_e32:
2636 case AMDGPU::V_MAC_F32_e32:
2637 case AMDGPU::V_FMAC_F32_e32: {
2638 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2639 AMDGPU::OpName::src0);
2640 const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2641 if (!Src0->isReg() && !Src0->isImm())
2644 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2651 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2652 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2653 const MachineOperand *Src0Mods =
2654 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2655 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2656 const MachineOperand *Src1Mods =
2657 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2658 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2659 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2660 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2662 if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2663 // If we have an SGPR input, we will violate the constant bus restriction.
2664 (ST.getConstantBusLimit(Opc) > 1 ||
2666 !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2667 if (auto Imm = getFoldableImm(Src2)) {
2669 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2670 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2671 if (pseudoToMCOpcode(NewOpc) != -1)
2672 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2679 IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2680 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2681 if (auto Imm = getFoldableImm(Src1)) {
2682 if (pseudoToMCOpcode(NewOpc) != -1)
2683 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2689 if (auto Imm = getFoldableImm(Src0)) {
2690 if (pseudoToMCOpcode(NewOpc) != -1 &&
2691 isOperandLegal(MI, AMDGPU::getNamedOperandIdx(NewOpc,
2692 AMDGPU::OpName::src0), Src1))
2693 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2701 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2702 : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2703 if (pseudoToMCOpcode(NewOpc) == -1)
2706 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2708 .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2710 .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2712 .addImm(0) // Src mods
2714 .addImm(Clamp ? Clamp->getImm() : 0)
2715 .addImm(Omod ? Omod->getImm() : 0);
2718 // It's not generally safe to move VALU instructions across these since it will
2719 // start using the register as a base index rather than directly.
2720 // XXX - Why isn't hasSideEffects sufficient for these?
2721 static bool changesVGPRIndexingMode(const MachineInstr &MI) {
2722 switch (MI.getOpcode()) {
2723 case AMDGPU::S_SET_GPR_IDX_ON:
2724 case AMDGPU::S_SET_GPR_IDX_MODE:
2725 case AMDGPU::S_SET_GPR_IDX_OFF:
2732 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2733 const MachineBasicBlock *MBB,
2734 const MachineFunction &MF) const {
2735 // XXX - Do we want the SP check in the base implementation?
2737 // Target-independent instructions do not have an implicit-use of EXEC, even
2738 // when they operate on VGPRs. Treating EXEC modifications as scheduling
2739 // boundaries prevents incorrect movements of such instructions.
2740 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2741 MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2742 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2743 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2744 MI.getOpcode() == AMDGPU::S_DENORM_MODE ||
2745 changesVGPRIndexingMode(MI);
2748 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2749 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2750 Opcode == AMDGPU::DS_GWS_INIT ||
2751 Opcode == AMDGPU::DS_GWS_SEMA_V ||
2752 Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2753 Opcode == AMDGPU::DS_GWS_SEMA_P ||
2754 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2755 Opcode == AMDGPU::DS_GWS_BARRIER;
2758 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
2759 unsigned Opcode = MI.getOpcode();
2761 if (MI.mayStore() && isSMRD(MI))
2762 return true; // scalar store or atomic
2764 // This will terminate the function when other lanes may need to continue.
2768 // These instructions cause shader I/O that may cause hardware lockups
2769 // when executed with an empty EXEC mask.
2771 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2772 // EXEC = 0, but checking for that case here seems not worth it
2773 // given the typical code patterns.
2774 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2775 Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2776 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2777 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2780 if (MI.isCall() || MI.isInlineAsm())
2781 return true; // conservative assumption
2783 // These are like SALU instructions in terms of effects, so it's questionable
2784 // whether we should return true for those.
2786 // However, executing them with EXEC = 0 causes them to operate on undefined
2787 // data, which we avoid by returning true here.
2788 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2794 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
2795 const MachineInstr &MI) const {
2796 if (MI.isMetaInstruction())
2799 // This won't read exec if this is an SGPR->SGPR copy.
2800 if (MI.isCopyLike()) {
2801 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2804 // Make sure this isn't copying exec as a normal operand
2805 return MI.readsRegister(AMDGPU::EXEC, &RI);
2808 // Make a conservative assumption about the callee.
2812 // Be conservative with any unhandled generic opcodes.
2813 if (!isTargetSpecificOpcode(MI.getOpcode()))
2816 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2819 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2820 switch (Imm.getBitWidth()) {
2821 case 1: // This likely will be a condition code mask.
2825 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
2826 ST.hasInv2PiInlineImm());
2828 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
2829 ST.hasInv2PiInlineImm());
2831 return ST.has16BitInsts() &&
2832 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
2833 ST.hasInv2PiInlineImm());
2835 llvm_unreachable("invalid bitwidth");
2839 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
2840 uint8_t OperandType) const {
2842 OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2843 OperandType > AMDGPU::OPERAND_SRC_LAST)
2846 // MachineOperand provides no way to tell the true operand size, since it only
2847 // records a 64-bit value. We need to know the size to determine if a 32-bit
2848 // floating point immediate bit pattern is legal for an integer immediate. It
2849 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2851 int64_t Imm = MO.getImm();
2852 switch (OperandType) {
2853 case AMDGPU::OPERAND_REG_IMM_INT32:
2854 case AMDGPU::OPERAND_REG_IMM_FP32:
2855 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2856 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2857 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2858 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: {
2859 int32_t Trunc = static_cast<int32_t>(Imm);
2860 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
2862 case AMDGPU::OPERAND_REG_IMM_INT64:
2863 case AMDGPU::OPERAND_REG_IMM_FP64:
2864 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2865 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2866 return AMDGPU::isInlinableLiteral64(MO.getImm(),
2867 ST.hasInv2PiInlineImm());
2868 case AMDGPU::OPERAND_REG_IMM_INT16:
2869 case AMDGPU::OPERAND_REG_IMM_FP16:
2870 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2871 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2872 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
2873 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
2874 if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2875 // A few special case instructions have 16-bit operands on subtargets
2876 // where 16-bit instructions are not legal.
2877 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2878 // constants in these cases
2879 int16_t Trunc = static_cast<int16_t>(Imm);
2880 return ST.has16BitInsts() &&
2881 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
2886 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2887 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2888 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2889 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2890 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
2891 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
2892 uint32_t Trunc = static_cast<uint32_t>(Imm);
2893 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm());
2896 llvm_unreachable("invalid bitwidth");
2900 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO,
2901 const MCOperandInfo &OpInfo) const {
2902 switch (MO.getType()) {
2903 case MachineOperand::MO_Register:
2905 case MachineOperand::MO_Immediate:
2906 return !isInlineConstant(MO, OpInfo);
2907 case MachineOperand::MO_FrameIndex:
2908 case MachineOperand::MO_MachineBasicBlock:
2909 case MachineOperand::MO_ExternalSymbol:
2910 case MachineOperand::MO_GlobalAddress:
2911 case MachineOperand::MO_MCSymbol:
2914 llvm_unreachable("unexpected operand type");
2918 static bool compareMachineOp(const MachineOperand &Op0,
2919 const MachineOperand &Op1) {
2920 if (Op0.getType() != Op1.getType())
2923 switch (Op0.getType()) {
2924 case MachineOperand::MO_Register:
2925 return Op0.getReg() == Op1.getReg();
2926 case MachineOperand::MO_Immediate:
2927 return Op0.getImm() == Op1.getImm();
2929 llvm_unreachable("Didn't expect to be comparing these operand types");
2933 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
2934 const MachineOperand &MO) const {
2935 const MCInstrDesc &InstDesc = MI.getDesc();
2936 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2938 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2940 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2943 if (OpInfo.RegClass < 0)
2946 const MachineFunction *MF = MI.getParent()->getParent();
2947 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2949 if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
2950 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
2951 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2952 AMDGPU::OpName::src2))
2954 return RI.opCanUseInlineConstant(OpInfo.OperandType);
2957 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2960 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2963 return ST.hasVOP3Literal();
2966 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2967 int Op32 = AMDGPU::getVOPe32(Opcode);
2971 return pseudoToMCOpcode(Op32) != -1;
2974 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2975 // The src0_modifier operand is present on all instructions
2976 // that have modifiers.
2978 return AMDGPU::getNamedOperandIdx(Opcode,
2979 AMDGPU::OpName::src0_modifiers) != -1;
2982 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
2983 unsigned OpName) const {
2984 const MachineOperand *Mods = getNamedOperand(MI, OpName);
2985 return Mods && Mods->getImm();
2988 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
2989 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2990 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2991 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2992 hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2993 hasModifiersSet(MI, AMDGPU::OpName::omod);
2996 bool SIInstrInfo::canShrink(const MachineInstr &MI,
2997 const MachineRegisterInfo &MRI) const {
2998 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2999 // Can't shrink instruction with three operands.
3000 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
3001 // a special case for it. It can only be shrunk if the third operand
3002 // is vcc, and src0_modifiers and src1_modifiers are not set.
3003 // We should handle this the same way we handle vopc, by addding
3004 // a register allocation hint pre-regalloc and then do the shrinking
3007 switch (MI.getOpcode()) {
3008 default: return false;
3010 case AMDGPU::V_ADDC_U32_e64:
3011 case AMDGPU::V_SUBB_U32_e64:
3012 case AMDGPU::V_SUBBREV_U32_e64: {
3013 const MachineOperand *Src1
3014 = getNamedOperand(MI, AMDGPU::OpName::src1);
3015 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
3017 // Additional verification is needed for sdst/src2.
3020 case AMDGPU::V_MAC_F32_e64:
3021 case AMDGPU::V_MAC_F16_e64:
3022 case AMDGPU::V_FMAC_F32_e64:
3023 case AMDGPU::V_FMAC_F16_e64:
3024 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
3025 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
3029 case AMDGPU::V_CNDMASK_B32_e64:
3034 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3035 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
3036 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
3039 // We don't need to check src0, all input types are legal, so just make sure
3040 // src0 isn't using any modifiers.
3041 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
3044 // Can it be shrunk to a valid 32 bit opcode?
3045 if (!hasVALU32BitEncoding(MI.getOpcode()))
3048 // Check output modifiers
3049 return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
3050 !hasModifiersSet(MI, AMDGPU::OpName::clamp);
3053 // Set VCC operand with all flags from \p Orig, except for setting it as
3055 static void copyFlagsToImplicitVCC(MachineInstr &MI,
3056 const MachineOperand &Orig) {
3058 for (MachineOperand &Use : MI.implicit_operands()) {
3059 if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
3060 Use.setIsUndef(Orig.isUndef());
3061 Use.setIsKill(Orig.isKill());
3067 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
3068 unsigned Op32) const {
3069 MachineBasicBlock *MBB = MI.getParent();;
3070 MachineInstrBuilder Inst32 =
3071 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
3073 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3074 // For VOPC instructions, this is replaced by an implicit def of vcc.
3075 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
3076 if (Op32DstIdx != -1) {
3078 Inst32.add(MI.getOperand(0));
3080 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3081 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3085 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3087 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3091 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3094 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3095 if (Op32Src2Idx != -1) {
3098 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3099 // replaced with an implicit read of vcc. This was already added
3100 // during the initial BuildMI, so find it to preserve the flags.
3101 copyFlagsToImplicitVCC(*Inst32, *Src2);
3108 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
3109 const MachineOperand &MO,
3110 const MCOperandInfo &OpInfo) const {
3111 // Literal constants use the constant bus.
3112 //if (isLiteralConstantLike(MO, OpInfo))
3115 return !isInlineConstant(MO, OpInfo);
3118 return true; // Misc other operands like FrameIndex
3123 if (Register::isVirtualRegister(MO.getReg()))
3124 return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3127 if (MO.getReg() == AMDGPU::SGPR_NULL)
3130 // SGPRs use the constant bus
3131 if (MO.isImplicit()) {
3132 return MO.getReg() == AMDGPU::M0 ||
3133 MO.getReg() == AMDGPU::VCC ||
3134 MO.getReg() == AMDGPU::VCC_LO;
3136 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3137 AMDGPU::SReg_64RegClass.contains(MO.getReg());
3141 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3142 for (const MachineOperand &MO : MI.implicit_operands()) {
3143 // We only care about reads.
3147 switch (MO.getReg()) {
3149 case AMDGPU::VCC_LO:
3150 case AMDGPU::VCC_HI:
3152 case AMDGPU::FLAT_SCR:
3160 return AMDGPU::NoRegister;
3163 static bool shouldReadExec(const MachineInstr &MI) {
3164 if (SIInstrInfo::isVALU(MI)) {
3165 switch (MI.getOpcode()) {
3166 case AMDGPU::V_READLANE_B32:
3167 case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3168 case AMDGPU::V_READLANE_B32_gfx10:
3169 case AMDGPU::V_READLANE_B32_vi:
3170 case AMDGPU::V_WRITELANE_B32:
3171 case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3172 case AMDGPU::V_WRITELANE_B32_gfx10:
3173 case AMDGPU::V_WRITELANE_B32_vi:
3180 if (MI.isPreISelOpcode() ||
3181 SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3182 SIInstrInfo::isSALU(MI) ||
3183 SIInstrInfo::isSMRD(MI))
3189 static bool isSubRegOf(const SIRegisterInfo &TRI,
3190 const MachineOperand &SuperVec,
3191 const MachineOperand &SubReg) {
3192 if (Register::isPhysicalRegister(SubReg.getReg()))
3193 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3195 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3196 SubReg.getReg() == SuperVec.getReg();
3199 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
3200 StringRef &ErrInfo) const {
3201 uint16_t Opcode = MI.getOpcode();
3202 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3205 const MachineFunction *MF = MI.getParent()->getParent();
3206 const MachineRegisterInfo &MRI = MF->getRegInfo();
3208 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3209 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3210 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3212 // Make sure the number of operands is correct.
3213 const MCInstrDesc &Desc = get(Opcode);
3214 if (!Desc.isVariadic() &&
3215 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3216 ErrInfo = "Instruction has wrong number of operands.";
3220 if (MI.isInlineAsm()) {
3221 // Verify register classes for inlineasm constraints.
3222 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3224 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3228 const MachineOperand &Op = MI.getOperand(I);
3232 Register Reg = Op.getReg();
3233 if (!Register::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3234 ErrInfo = "inlineasm operand has incorrect register class.";
3242 // Make sure the register classes are correct.
3243 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3244 if (MI.getOperand(i).isFPImm()) {
3245 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3246 "all fp values to integers.";
3250 int RegClass = Desc.OpInfo[i].RegClass;
3252 switch (Desc.OpInfo[i].OperandType) {
3253 case MCOI::OPERAND_REGISTER:
3254 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3255 ErrInfo = "Illegal immediate value for operand.";
3259 case AMDGPU::OPERAND_REG_IMM_INT32:
3260 case AMDGPU::OPERAND_REG_IMM_FP32:
3262 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
3263 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
3264 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
3265 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
3266 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
3267 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
3268 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
3269 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
3270 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
3271 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
3272 const MachineOperand &MO = MI.getOperand(i);
3273 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3274 ErrInfo = "Illegal immediate value for operand.";
3279 case MCOI::OPERAND_IMMEDIATE:
3280 case AMDGPU::OPERAND_KIMM32:
3281 // Check if this operand is an immediate.
3282 // FrameIndex operands will be replaced by immediates, so they are
3284 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3285 ErrInfo = "Expected immediate, but got non-immediate";
3293 if (!MI.getOperand(i).isReg())
3296 if (RegClass != -1) {
3297 Register Reg = MI.getOperand(i).getReg();
3298 if (Reg == AMDGPU::NoRegister || Register::isVirtualRegister(Reg))
3301 const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3302 if (!RC->contains(Reg)) {
3303 ErrInfo = "Operand has incorrect register class.";
3311 if (!ST.hasSDWA()) {
3312 ErrInfo = "SDWA is not supported on this target";
3316 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3318 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3320 for (int OpIdx: OpIndicies) {
3323 const MachineOperand &MO = MI.getOperand(OpIdx);
3325 if (!ST.hasSDWAScalar()) {
3327 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3328 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3332 // No immediates on GFX9
3334 ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3340 if (!ST.hasSDWAOmod()) {
3341 // No omod allowed on VI
3342 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3343 if (OMod != nullptr &&
3344 (!OMod->isImm() || OMod->getImm() != 0)) {
3345 ErrInfo = "OMod not allowed in SDWA instructions on VI";
3350 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3351 if (isVOPC(BasicOpcode)) {
3352 if (!ST.hasSDWASdst() && DstIdx != -1) {
3353 // Only vcc allowed as dst on VI for VOPC
3354 const MachineOperand &Dst = MI.getOperand(DstIdx);
3355 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3356 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3359 } else if (!ST.hasSDWAOutModsVOPC()) {
3360 // No clamp allowed on GFX9 for VOPC
3361 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3362 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3363 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3367 // No omod allowed on GFX9 for VOPC
3368 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3369 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3370 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3376 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3377 if (DstUnused && DstUnused->isImm() &&
3378 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3379 const MachineOperand &Dst = MI.getOperand(DstIdx);
3380 if (!Dst.isReg() || !Dst.isTied()) {
3381 ErrInfo = "Dst register should have tied register";
3385 const MachineOperand &TiedMO =
3386 MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3387 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3389 "Dst register should be tied to implicit use of preserved register";
3391 } else if (Register::isPhysicalRegister(TiedMO.getReg()) &&
3392 Dst.getReg() != TiedMO.getReg()) {
3393 ErrInfo = "Dst register should use same physical register as preserved";
3400 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3401 // Ensure that the return type used is large enough for all the options
3402 // being used TFE/LWE require an extra result register.
3403 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3405 uint64_t DMaskImm = DMask->getImm();
3407 isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3408 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3409 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3410 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3412 // Adjust for packed 16 bit values
3413 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3416 // Adjust if using LWE or TFE
3417 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3420 const uint32_t DstIdx =
3421 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3422 const MachineOperand &Dst = MI.getOperand(DstIdx);
3424 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3425 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3426 if (RegCount > DstSize) {
3427 ErrInfo = "MIMG instruction returns too many registers for dst "
3435 // Verify VOP*. Ignore multiple sgpr operands on writelane.
3436 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3437 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3438 // Only look at the true operands. Only a real operand can use the constant
3439 // bus, and we don't want to check pseudo-operands like the source modifier
3441 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3443 unsigned ConstantBusCount = 0;
3444 unsigned LiteralCount = 0;
3446 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3449 SmallVector<unsigned, 2> SGPRsUsed;
3450 unsigned SGPRUsed = findImplicitSGPRRead(MI);
3451 if (SGPRUsed != AMDGPU::NoRegister) {
3453 SGPRsUsed.push_back(SGPRUsed);
3456 for (int OpIdx : OpIndices) {
3459 const MachineOperand &MO = MI.getOperand(OpIdx);
3460 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3462 SGPRUsed = MO.getReg();
3463 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3464 return !RI.regsOverlap(SGPRUsed, SGPR);
3467 SGPRsUsed.push_back(SGPRUsed);
3475 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3476 // v_writelane_b32 is an exception from constant bus restriction:
3477 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3478 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3479 Opcode != AMDGPU::V_WRITELANE_B32) {
3480 ErrInfo = "VOP* instruction violates constant bus restriction";
3484 if (isVOP3(MI) && LiteralCount) {
3485 if (LiteralCount && !ST.hasVOP3Literal()) {
3486 ErrInfo = "VOP3 instruction uses literal";
3489 if (LiteralCount > 1) {
3490 ErrInfo = "VOP3 instruction uses more than one literal";
3496 // Special case for writelane - this can break the multiple constant bus rule,
3497 // but still can't use more than one SGPR register
3498 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
3499 unsigned SGPRCount = 0;
3500 Register SGPRUsed = AMDGPU::NoRegister;
3502 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx}) {
3506 const MachineOperand &MO = MI.getOperand(OpIdx);
3508 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3509 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
3510 if (MO.getReg() != SGPRUsed)
3512 SGPRUsed = MO.getReg();
3515 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
3516 ErrInfo = "WRITELANE instruction violates constant bus restriction";
3522 // Verify misc. restrictions on specific instructions.
3523 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3524 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3525 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3526 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3527 const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3528 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3529 if (!compareMachineOp(Src0, Src1) &&
3530 !compareMachineOp(Src0, Src2)) {
3531 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3537 if (isSOP2(MI) || isSOPC(MI)) {
3538 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3539 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3540 unsigned Immediates = 0;
3542 if (!Src0.isReg() &&
3543 !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3545 if (!Src1.isReg() &&
3546 !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3549 if (Immediates > 1) {
3550 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3556 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3557 if (Desc.isBranch()) {
3559 ErrInfo = "invalid branch target for SOPK instruction";
3563 uint64_t Imm = Op->getImm();
3564 if (sopkIsZext(MI)) {
3565 if (!isUInt<16>(Imm)) {
3566 ErrInfo = "invalid immediate for SOPK instruction";
3570 if (!isInt<16>(Imm)) {
3571 ErrInfo = "invalid immediate for SOPK instruction";
3578 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3579 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3580 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3581 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3582 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3583 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3585 const unsigned StaticNumOps = Desc.getNumOperands() +
3586 Desc.getNumImplicitUses();
3587 const unsigned NumImplicitOps = IsDst ? 2 : 1;
3589 // Allow additional implicit operands. This allows a fixup done by the post
3590 // RA scheduler where the main implicit operand is killed and implicit-defs
3591 // are added for sub-registers that remain live after this instruction.
3592 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3593 ErrInfo = "missing implicit register operands";
3597 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3599 if (!Dst->isUse()) {
3600 ErrInfo = "v_movreld_b32 vdst should be a use operand";
3605 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3606 UseOpIdx != StaticNumOps + 1) {
3607 ErrInfo = "movrel implicit operands should be tied";
3612 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3613 const MachineOperand &ImpUse
3614 = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3615 if (!ImpUse.isReg() || !ImpUse.isUse() ||
3616 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3617 ErrInfo = "src0 should be subreg of implicit vector use";
3622 // Make sure we aren't losing exec uses in the td files. This mostly requires
3623 // being careful when using let Uses to try to add other use registers.
3624 if (shouldReadExec(MI)) {
3625 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3626 ErrInfo = "VALU instruction does not implicitly read exec mask";
3632 if (MI.mayStore()) {
3633 // The register offset form of scalar stores may only use m0 as the
3634 // soffset register.
3635 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3636 if (Soff && Soff->getReg() != AMDGPU::M0) {
3637 ErrInfo = "scalar stores must use m0 as offset register";
3643 if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3644 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3645 if (Offset->getImm() != 0) {
3646 ErrInfo = "subtarget does not support offsets in flat instructions";
3652 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3654 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3655 AMDGPU::OpName::vaddr0);
3656 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3657 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3658 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3659 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
3660 const AMDGPU::MIMGDimInfo *Dim =
3661 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm());
3664 ErrInfo = "dim is out of range";
3668 bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3669 unsigned AddrWords = BaseOpcode->NumExtraArgs +
3670 (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3671 (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3672 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3674 unsigned VAddrWords;
3676 VAddrWords = SRsrcIdx - VAddr0Idx;
3678 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3679 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3682 else if (AddrWords > 4)
3684 else if (AddrWords == 3 && VAddrWords == 4) {
3685 // CodeGen uses the V4 variant of instructions for three addresses,
3686 // because the selection DAG does not support non-power-of-two types.
3691 if (VAddrWords != AddrWords) {
3692 ErrInfo = "bad vaddr size";
3698 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3700 using namespace AMDGPU::DPP;
3702 unsigned DC = DppCt->getImm();
3703 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3704 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3705 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
3706 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
3707 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
3708 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
3709 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
3710 ErrInfo = "Invalid dpp_ctrl value";
3713 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3714 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3715 ErrInfo = "Invalid dpp_ctrl value: "
3716 "wavefront shifts are not supported on GFX10+";
3719 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3720 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3721 ErrInfo = "Invalid dpp_ctrl value: "
3722 "broadcasts are not supported on GFX10+";
3725 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
3726 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3727 ErrInfo = "Invalid dpp_ctrl value: "
3728 "row_share and row_xmask are not supported before GFX10";
3736 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3737 switch (MI.getOpcode()) {
3738 default: return AMDGPU::INSTRUCTION_LIST_END;
3739 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3740 case AMDGPU::COPY: return AMDGPU::COPY;
3741 case AMDGPU::PHI: return AMDGPU::PHI;
3742 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3743 case AMDGPU::WQM: return AMDGPU::WQM;
3744 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
3745 case AMDGPU::WWM: return AMDGPU::WWM;
3746 case AMDGPU::S_MOV_B32: {
3747 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3748 return MI.getOperand(1).isReg() ||
3749 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
3750 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3752 case AMDGPU::S_ADD_I32:
3753 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3754 case AMDGPU::S_ADDC_U32:
3755 return AMDGPU::V_ADDC_U32_e32;
3756 case AMDGPU::S_SUB_I32:
3757 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3758 // FIXME: These are not consistently handled, and selected when the carry is
3760 case AMDGPU::S_ADD_U32:
3761 return AMDGPU::V_ADD_I32_e32;
3762 case AMDGPU::S_SUB_U32:
3763 return AMDGPU::V_SUB_I32_e32;
3764 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3765 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3766 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3767 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3768 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3769 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3770 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3771 case AMDGPU::S_XNOR_B32:
3772 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3773 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3774 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3775 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3776 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3777 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3778 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3779 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3780 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3781 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3782 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3783 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3784 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3785 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3786 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3787 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3788 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3789 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3790 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3791 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3792 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3793 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3794 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3795 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3796 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3797 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3798 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3799 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3800 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3801 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3802 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3803 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3804 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3805 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3806 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3807 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3808 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3809 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3810 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3813 "Unexpected scalar opcode without corresponding vector one!");
3816 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
3817 unsigned OpNo) const {
3818 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3819 const MCInstrDesc &Desc = get(MI.getOpcode());
3820 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3821 Desc.OpInfo[OpNo].RegClass == -1) {
3822 Register Reg = MI.getOperand(OpNo).getReg();
3824 if (Register::isVirtualRegister(Reg))
3825 return MRI.getRegClass(Reg);
3826 return RI.getPhysRegClass(Reg);
3829 unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3830 return RI.getRegClass(RCID);
3833 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3834 MachineBasicBlock::iterator I = MI;
3835 MachineBasicBlock *MBB = MI.getParent();
3836 MachineOperand &MO = MI.getOperand(OpIdx);
3837 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
3838 const SIRegisterInfo *TRI =
3839 static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3840 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3841 const TargetRegisterClass *RC = RI.getRegClass(RCID);
3842 unsigned Size = TRI->getRegSizeInBits(*RC);
3843 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3845 Opcode = AMDGPU::COPY;
3846 else if (RI.isSGPRClass(RC))
3847 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3849 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3850 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3851 VRC = &AMDGPU::VReg_64RegClass;
3853 VRC = &AMDGPU::VGPR_32RegClass;
3855 Register Reg = MRI.createVirtualRegister(VRC);
3856 DebugLoc DL = MBB->findDebugLoc(I);
3857 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3858 MO.ChangeToRegister(Reg, false);
3861 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI,
3862 MachineRegisterInfo &MRI,
3863 MachineOperand &SuperReg,
3864 const TargetRegisterClass *SuperRC,
3866 const TargetRegisterClass *SubRC)
3868 MachineBasicBlock *MBB = MI->getParent();
3869 DebugLoc DL = MI->getDebugLoc();
3870 Register SubReg = MRI.createVirtualRegister(SubRC);
3872 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3873 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3874 .addReg(SuperReg.getReg(), 0, SubIdx);
3878 // Just in case the super register is itself a sub-register, copy it to a new
3879 // value so we don't need to worry about merging its subreg index with the
3880 // SubIdx passed to this function. The register coalescer should be able to
3881 // eliminate this extra copy.
3882 Register NewSuperReg = MRI.createVirtualRegister(SuperRC);
3884 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3885 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3887 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3888 .addReg(NewSuperReg, 0, SubIdx);
3893 MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
3894 MachineBasicBlock::iterator MII,
3895 MachineRegisterInfo &MRI,
3897 const TargetRegisterClass *SuperRC,
3899 const TargetRegisterClass *SubRC) const {
3901 if (SubIdx == AMDGPU::sub0)
3902 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3903 if (SubIdx == AMDGPU::sub1)
3904 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3906 llvm_unreachable("Unhandled register index for immediate");
3909 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3911 return MachineOperand::CreateReg(SubReg, false);
3914 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3915 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3916 assert(Inst.getNumExplicitOperands() == 3);
3917 MachineOperand Op1 = Inst.getOperand(1);
3918 Inst.RemoveOperand(1);
3919 Inst.addOperand(Op1);
3922 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
3923 const MCOperandInfo &OpInfo,
3924 const MachineOperand &MO) const {
3928 Register Reg = MO.getReg();
3929 const TargetRegisterClass *RC = Register::isVirtualRegister(Reg)
3930 ? MRI.getRegClass(Reg)
3931 : RI.getPhysRegClass(Reg);
3933 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass);
3934 if (MO.getSubReg()) {
3935 const MachineFunction *MF = MO.getParent()->getParent()->getParent();
3936 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF);
3940 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg());
3944 return RC->hasSuperClassEq(DRC);
3947 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
3948 const MCOperandInfo &OpInfo,
3949 const MachineOperand &MO) const {
3951 return isLegalRegOperand(MRI, OpInfo, MO);
3953 // Handle non-register types that are treated like immediates.
3954 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3958 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3959 const MachineOperand *MO) const {
3960 const MachineFunction &MF = *MI.getParent()->getParent();
3961 const MachineRegisterInfo &MRI = MF.getRegInfo();
3962 const MCInstrDesc &InstDesc = MI.getDesc();
3963 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3964 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3965 const TargetRegisterClass *DefinedRC =
3966 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3968 MO = &MI.getOperand(OpIdx);
3970 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3971 int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3972 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3973 if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3976 SmallDenseSet<RegSubRegPair> SGPRsUsed;
3978 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3980 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3983 const MachineOperand &Op = MI.getOperand(i);
3985 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3986 if (!SGPRsUsed.count(SGPR) &&
3987 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3988 if (--ConstantBusLimit <= 0)
3990 SGPRsUsed.insert(SGPR);
3992 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3993 if (--ConstantBusLimit <= 0)
3995 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3996 isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3997 if (!VOP3LiteralLimit--)
3999 if (--ConstantBusLimit <= 0)
4007 return isLegalRegOperand(MRI, OpInfo, *MO);
4010 // Handle non-register types that are treated like immediates.
4011 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
4014 // This operand expects an immediate.
4018 return isImmOperandLegal(MI, OpIdx, *MO);
4021 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
4022 MachineInstr &MI) const {
4023 unsigned Opc = MI.getOpcode();
4024 const MCInstrDesc &InstrDesc = get(Opc);
4026 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4027 MachineOperand &Src0 = MI.getOperand(Src0Idx);
4029 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
4030 MachineOperand &Src1 = MI.getOperand(Src1Idx);
4032 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
4033 // we need to only have one constant bus use before GFX10.
4034 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
4035 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
4036 Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
4037 isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
4038 legalizeOpWithMove(MI, Src0Idx);
4040 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
4041 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
4042 // src0/src1 with V_READFIRSTLANE.
4043 if (Opc == AMDGPU::V_WRITELANE_B32) {
4044 const DebugLoc &DL = MI.getDebugLoc();
4045 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
4046 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4047 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4049 Src0.ChangeToRegister(Reg, false);
4051 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
4052 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4053 const DebugLoc &DL = MI.getDebugLoc();
4054 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4056 Src1.ChangeToRegister(Reg, false);
4061 // No VOP2 instructions support AGPRs.
4062 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
4063 legalizeOpWithMove(MI, Src0Idx);
4065 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
4066 legalizeOpWithMove(MI, Src1Idx);
4068 // VOP2 src0 instructions support all operand types, so we don't need to check
4069 // their legality. If src1 is already legal, we don't need to do anything.
4070 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
4073 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
4074 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
4075 // select is uniform.
4076 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
4077 RI.isVGPR(MRI, Src1.getReg())) {
4078 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4079 const DebugLoc &DL = MI.getDebugLoc();
4080 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4082 Src1.ChangeToRegister(Reg, false);
4086 // We do not use commuteInstruction here because it is too aggressive and will
4087 // commute if it is possible. We only want to commute here if it improves
4088 // legality. This can be called a fairly large number of times so don't waste
4089 // compile time pointlessly swapping and checking legality again.
4090 if (HasImplicitSGPR || !MI.isCommutable()) {
4091 legalizeOpWithMove(MI, Src1Idx);
4095 // If src0 can be used as src1, commuting will make the operands legal.
4096 // Otherwise we have to give up and insert a move.
4098 // TODO: Other immediate-like operand kinds could be commuted if there was a
4099 // MachineOperand::ChangeTo* for them.
4100 if ((!Src1.isImm() && !Src1.isReg()) ||
4101 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
4102 legalizeOpWithMove(MI, Src1Idx);
4106 int CommutedOpc = commuteOpcode(MI);
4107 if (CommutedOpc == -1) {
4108 legalizeOpWithMove(MI, Src1Idx);
4112 MI.setDesc(get(CommutedOpc));
4114 Register Src0Reg = Src0.getReg();
4115 unsigned Src0SubReg = Src0.getSubReg();
4116 bool Src0Kill = Src0.isKill();
4119 Src0.ChangeToImmediate(Src1.getImm());
4120 else if (Src1.isReg()) {
4121 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4122 Src0.setSubReg(Src1.getSubReg());
4124 llvm_unreachable("Should only have register or immediate operands");
4126 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4127 Src1.setSubReg(Src0SubReg);
4128 fixImplicitOperands(MI);
4131 // Legalize VOP3 operands. All operand types are supported for any operand
4132 // but only one literal constant and only starting from GFX10.
4133 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
4134 MachineInstr &MI) const {
4135 unsigned Opc = MI.getOpcode();
4138 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4139 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4140 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4143 if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4144 Opc == AMDGPU::V_PERMLANEX16_B32) {
4145 // src1 and src2 must be scalar
4146 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4147 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4148 const DebugLoc &DL = MI.getDebugLoc();
4149 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4150 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4151 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4153 Src1.ChangeToRegister(Reg, false);
4155 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4156 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4157 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4159 Src2.ChangeToRegister(Reg, false);
4163 // Find the one SGPR operand we are allowed to use.
4164 int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4165 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4166 SmallDenseSet<unsigned> SGPRsUsed;
4167 unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4168 if (SGPRReg != AMDGPU::NoRegister) {
4169 SGPRsUsed.insert(SGPRReg);
4173 for (unsigned i = 0; i < 3; ++i) {
4174 int Idx = VOP3Idx[i];
4177 MachineOperand &MO = MI.getOperand(Idx);
4180 if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4183 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4191 legalizeOpWithMove(MI, Idx);
4195 if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4196 !isOperandLegal(MI, Idx, &MO)) {
4197 legalizeOpWithMove(MI, Idx);
4201 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4202 continue; // VGPRs are legal
4204 // We can use one SGPR in each VOP3 instruction prior to GFX10
4205 // and two starting from GFX10.
4206 if (SGPRsUsed.count(MO.getReg()))
4208 if (ConstantBusLimit > 0) {
4209 SGPRsUsed.insert(MO.getReg());
4214 // If we make it this far, then the operand is not legal and we must
4216 legalizeOpWithMove(MI, Idx);
4220 unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
4221 MachineRegisterInfo &MRI) const {
4222 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4223 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4224 Register DstReg = MRI.createVirtualRegister(SRC);
4225 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4227 if (RI.hasAGPRs(VRC)) {
4228 VRC = RI.getEquivalentVGPRClass(VRC);
4229 Register NewSrcReg = MRI.createVirtualRegister(VRC);
4230 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4231 get(TargetOpcode::COPY), NewSrcReg)
4237 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4238 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4243 SmallVector<unsigned, 8> SRegs;
4244 for (unsigned i = 0; i < SubRegs; ++i) {
4245 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4246 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4247 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4248 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4249 SRegs.push_back(SGPR);
4252 MachineInstrBuilder MIB =
4253 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4254 get(AMDGPU::REG_SEQUENCE), DstReg);
4255 for (unsigned i = 0; i < SubRegs; ++i) {
4256 MIB.addReg(SRegs[i]);
4257 MIB.addImm(RI.getSubRegFromChannel(i));
4262 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
4263 MachineInstr &MI) const {
4265 // If the pointer is store in VGPRs, then we need to move them to
4266 // SGPRs using v_readfirstlane. This is safe because we only select
4267 // loads with uniform pointers to SMRD instruction so we know the
4268 // pointer value is uniform.
4269 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4270 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4271 unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4272 SBase->setReg(SGPR);
4274 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4275 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4276 unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4281 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
4282 MachineBasicBlock::iterator I,
4283 const TargetRegisterClass *DstRC,
4285 MachineRegisterInfo &MRI,
4286 const DebugLoc &DL) const {
4287 Register OpReg = Op.getReg();
4288 unsigned OpSubReg = Op.getSubReg();
4290 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4291 RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4293 // Check if operand is already the correct register class.
4297 Register DstReg = MRI.createVirtualRegister(DstRC);
4298 MachineInstr *Copy =
4299 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4304 MachineInstr *Def = MRI.getVRegDef(OpReg);
4308 // Try to eliminate the copy if it is copying an immediate value.
4309 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
4310 FoldImmediate(*Copy, *Def, OpReg, &MRI);
4312 bool ImpDef = Def->isImplicitDef();
4313 while (!ImpDef && Def && Def->isCopy()) {
4314 if (Def->getOperand(1).getReg().isPhysical())
4316 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
4317 ImpDef = Def && Def->isImplicitDef();
4319 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
4321 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
4324 // Emit the actual waterfall loop, executing the wrapped instruction for each
4325 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4326 // iteration, in the worst case we execute 64 (once per lane).
4328 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
4329 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4330 const DebugLoc &DL, MachineOperand &Rsrc) {
4331 MachineFunction &MF = *OrigBB.getParent();
4332 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4333 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4334 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4335 unsigned SaveExecOpc =
4336 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4337 unsigned XorTermOpc =
4338 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4340 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4341 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4343 MachineBasicBlock::iterator I = LoopBB.begin();
4345 Register VRsrc = Rsrc.getReg();
4346 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4348 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4349 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4350 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4351 Register AndCond = MRI.createVirtualRegister(BoolXExecRC);
4352 Register SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4353 Register SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4354 Register SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4355 Register SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4356 Register SRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4358 // Beginning of the loop, read the next Rsrc variant.
4359 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4360 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4361 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4362 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4363 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4364 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4365 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4366 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4368 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4370 .addImm(AMDGPU::sub0)
4372 .addImm(AMDGPU::sub1)
4374 .addImm(AMDGPU::sub2)
4376 .addImm(AMDGPU::sub3);
4378 // Update Rsrc operand to use the SGPR Rsrc.
4380 Rsrc.setIsKill(true);
4382 // Identify all lanes with identical Rsrc operands in their VGPRs.
4383 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4384 .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4385 .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4386 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4387 .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4388 .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4389 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4393 MRI.setSimpleHint(SaveExec, AndCond);
4395 // Update EXEC to matching lanes, saving original to SaveExec.
4396 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4397 .addReg(AndCond, RegState::Kill);
4399 // The original instruction is here; we insert the terminators after it.
4402 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4403 BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4406 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4409 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4410 // with SGPRs by iterating over all unique values across all lanes.
4411 static void loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
4412 MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4413 MachineBasicBlock &MBB = *MI.getParent();
4414 MachineFunction &MF = *MBB.getParent();
4415 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4416 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4417 MachineRegisterInfo &MRI = MF.getRegInfo();
4418 MachineBasicBlock::iterator I(&MI);
4419 const DebugLoc &DL = MI.getDebugLoc();
4420 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4421 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4422 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4424 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4426 // Save the EXEC mask
4427 BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4429 // Killed uses in the instruction we are waterfalling around will be
4430 // incorrect due to the added control-flow.
4431 for (auto &MO : MI.uses()) {
4432 if (MO.isReg() && MO.isUse()) {
4433 MRI.clearKillFlags(MO.getReg());
4437 // To insert the loop we need to split the block. Move everything after this
4438 // point to a new block, and insert a new empty block between the two.
4439 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
4440 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4441 MachineFunction::iterator MBBI(MBB);
4444 MF.insert(MBBI, LoopBB);
4445 MF.insert(MBBI, RemainderBB);
4447 LoopBB->addSuccessor(LoopBB);
4448 LoopBB->addSuccessor(RemainderBB);
4450 // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4451 MachineBasicBlock::iterator J = I++;
4452 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4453 RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4454 LoopBB->splice(LoopBB->begin(), &MBB, J);
4456 MBB.addSuccessor(LoopBB);
4458 // Update dominators. We know that MBB immediately dominates LoopBB, that
4459 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4460 // dominates all of the successors transferred to it from MBB that MBB used
4461 // to properly dominate.
4463 MDT->addNewBlock(LoopBB, &MBB);
4464 MDT->addNewBlock(RemainderBB, LoopBB);
4465 for (auto &Succ : RemainderBB->successors()) {
4466 if (MDT->properlyDominates(&MBB, Succ)) {
4467 MDT->changeImmediateDominator(Succ, RemainderBB);
4472 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4474 // Restore the EXEC mask
4475 MachineBasicBlock::iterator First = RemainderBB->begin();
4476 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4479 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4480 static std::tuple<unsigned, unsigned>
4481 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
4482 MachineBasicBlock &MBB = *MI.getParent();
4483 MachineFunction &MF = *MBB.getParent();
4484 MachineRegisterInfo &MRI = MF.getRegInfo();
4486 // Extract the ptr from the resource descriptor.
4488 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4489 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4491 // Create an empty resource descriptor
4492 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4493 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4494 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4495 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4496 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4499 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4502 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4503 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4504 .addImm(RsrcDataFormat & 0xFFFFFFFF);
4506 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4507 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4508 .addImm(RsrcDataFormat >> 32);
4510 // NewSRsrc = {Zero64, SRsrcFormat}
4511 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4513 .addImm(AMDGPU::sub0_sub1)
4514 .addReg(SRsrcFormatLo)
4515 .addImm(AMDGPU::sub2)
4516 .addReg(SRsrcFormatHi)
4517 .addImm(AMDGPU::sub3);
4519 return std::make_tuple(RsrcPtr, NewSRsrc);
4522 void SIInstrInfo::legalizeOperands(MachineInstr &MI,
4523 MachineDominatorTree *MDT) const {
4524 MachineFunction &MF = *MI.getParent()->getParent();
4525 MachineRegisterInfo &MRI = MF.getRegInfo();
4528 if (isVOP2(MI) || isVOPC(MI)) {
4529 legalizeOperandsVOP2(MRI, MI);
4535 legalizeOperandsVOP3(MRI, MI);
4541 legalizeOperandsSMRD(MRI, MI);
4545 // Legalize REG_SEQUENCE and PHI
4546 // The register class of the operands much be the same type as the register
4547 // class of the output.
4548 if (MI.getOpcode() == AMDGPU::PHI) {
4549 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4550 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4551 if (!MI.getOperand(i).isReg() ||
4552 !Register::isVirtualRegister(MI.getOperand(i).getReg()))
4554 const TargetRegisterClass *OpRC =
4555 MRI.getRegClass(MI.getOperand(i).getReg());
4556 if (RI.hasVectorRegisters(OpRC)) {
4563 // If any of the operands are VGPR registers, then they all most be
4564 // otherwise we will create illegal VGPR->SGPR copies when legalizing
4566 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4569 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) {
4570 VRC = &AMDGPU::VReg_1RegClass;
4572 VRC = RI.hasAGPRs(getOpRegClass(MI, 0))
4573 ? RI.getEquivalentAGPRClass(SRC)
4574 : RI.getEquivalentVGPRClass(SRC);
4576 VRC = RI.hasAGPRs(getOpRegClass(MI, 0))
4577 ? RI.getEquivalentAGPRClass(VRC)
4578 : RI.getEquivalentVGPRClass(VRC);
4585 // Update all the operands so they have the same type.
4586 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4587 MachineOperand &Op = MI.getOperand(I);
4588 if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4591 // MI is a PHI instruction.
4592 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4593 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4595 // Avoid creating no-op copies with the same src and dst reg class. These
4596 // confuse some of the machine passes.
4597 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4601 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4602 // VGPR dest type and SGPR sources, insert copies so all operands are
4603 // VGPRs. This seems to help operand folding / the register coalescer.
4604 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4605 MachineBasicBlock *MBB = MI.getParent();
4606 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4607 if (RI.hasVGPRs(DstRC)) {
4608 // Update all the operands so they are VGPR register classes. These may
4609 // not be the same register class because REG_SEQUENCE supports mixing
4610 // subregister index types e.g. sub0_sub1 + sub2 + sub3
4611 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4612 MachineOperand &Op = MI.getOperand(I);
4613 if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4616 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4617 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4621 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4629 // Legalize INSERT_SUBREG
4630 // src0 must have the same register class as dst
4631 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4632 Register Dst = MI.getOperand(0).getReg();
4633 Register Src0 = MI.getOperand(1).getReg();
4634 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4635 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4636 if (DstRC != Src0RC) {
4637 MachineBasicBlock *MBB = MI.getParent();
4638 MachineOperand &Op = MI.getOperand(1);
4639 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4644 // Legalize SI_INIT_M0
4645 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4646 MachineOperand &Src = MI.getOperand(0);
4647 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4648 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4652 // Legalize MIMG and MUBUF/MTBUF for shaders.
4654 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4655 // scratch memory access. In both cases, the legalization never involves
4656 // conversion to the addr64 form.
4658 (AMDGPU::isShader(MF.getFunction().getCallingConv()) &&
4659 (isMUBUF(MI) || isMTBUF(MI)))) {
4660 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4661 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4662 unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4663 SRsrc->setReg(SGPR);
4666 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4667 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4668 unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4669 SSamp->setReg(SGPR);
4674 // Legalize MUBUF* instructions.
4676 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4677 if (RsrcIdx != -1) {
4678 // We have an MUBUF instruction
4679 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4680 unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4681 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4682 RI.getRegClass(RsrcRC))) {
4683 // The operands are legal.
4684 // FIXME: We may need to legalize operands besided srsrc.
4688 // Legalize a VGPR Rsrc.
4690 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4691 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4692 // a zero-value SRsrc.
4694 // If the instruction is _OFFSET (both idxen and offen disabled), and we
4695 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4698 // Otherwise we are on non-ADDR64 hardware, and/or we have
4699 // idxen/offen/bothen and we fall back to a waterfall loop.
4701 MachineBasicBlock &MBB = *MI.getParent();
4703 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4704 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4705 // This is already an ADDR64 instruction so we need to add the pointer
4706 // extracted from the resource descriptor to the current value of VAddr.
4707 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4708 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4709 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4711 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4712 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4713 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4715 unsigned RsrcPtr, NewSRsrc;
4716 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4718 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4719 const DebugLoc &DL = MI.getDebugLoc();
4720 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4722 .addReg(RsrcPtr, 0, AMDGPU::sub0)
4723 .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4726 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4727 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4728 .addDef(CondReg1, RegState::Dead)
4729 .addReg(RsrcPtr, 0, AMDGPU::sub1)
4730 .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4731 .addReg(CondReg0, RegState::Kill)
4734 // NewVaddr = {NewVaddrHi, NewVaddrLo}
4735 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4737 .addImm(AMDGPU::sub0)
4739 .addImm(AMDGPU::sub1);
4741 VAddr->setReg(NewVAddr);
4742 Rsrc->setReg(NewSRsrc);
4743 } else if (!VAddr && ST.hasAddr64()) {
4744 // This instructions is the _OFFSET variant, so we need to convert it to
4746 assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4747 < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
4748 "FIXME: Need to emit flat atomics here");
4750 unsigned RsrcPtr, NewSRsrc;
4751 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4753 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4754 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4755 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4756 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4757 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4759 // Atomics rith return have have an additional tied operand and are
4760 // missing some of the special bits.
4761 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4762 MachineInstr *Addr64;
4765 // Regular buffer load / store.
4766 MachineInstrBuilder MIB =
4767 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4774 // Atomics do not have this operand.
4775 if (const MachineOperand *GLC =
4776 getNamedOperand(MI, AMDGPU::OpName::glc)) {
4777 MIB.addImm(GLC->getImm());
4779 if (const MachineOperand *DLC =
4780 getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4781 MIB.addImm(DLC->getImm());
4784 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4786 if (const MachineOperand *TFE =
4787 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4788 MIB.addImm(TFE->getImm());
4791 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
4793 MIB.cloneMemRefs(MI);
4796 // Atomics with return.
4797 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4804 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4808 MI.removeFromParent();
4810 // NewVaddr = {NewVaddrHi, NewVaddrLo}
4811 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4813 .addReg(RsrcPtr, 0, AMDGPU::sub0)
4814 .addImm(AMDGPU::sub0)
4815 .addReg(RsrcPtr, 0, AMDGPU::sub1)
4816 .addImm(AMDGPU::sub1);
4818 // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4820 loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4825 void SIInstrInfo::moveToVALU(MachineInstr &TopInst,
4826 MachineDominatorTree *MDT) const {
4827 SetVectorType Worklist;
4828 Worklist.insert(&TopInst);
4830 while (!Worklist.empty()) {
4831 MachineInstr &Inst = *Worklist.pop_back_val();
4832 MachineBasicBlock *MBB = Inst.getParent();
4833 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
4835 unsigned Opcode = Inst.getOpcode();
4836 unsigned NewOpcode = getVALUOp(Inst);
4838 // Handle some special cases
4842 case AMDGPU::S_ADD_U64_PSEUDO:
4843 case AMDGPU::S_SUB_U64_PSEUDO:
4844 splitScalar64BitAddSub(Worklist, Inst, MDT);
4845 Inst.eraseFromParent();
4847 case AMDGPU::S_ADD_I32:
4848 case AMDGPU::S_SUB_I32:
4849 // FIXME: The u32 versions currently selected use the carry.
4850 if (moveScalarAddSub(Worklist, Inst, MDT))
4855 case AMDGPU::S_AND_B64:
4856 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4857 Inst.eraseFromParent();
4860 case AMDGPU::S_OR_B64:
4861 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4862 Inst.eraseFromParent();
4865 case AMDGPU::S_XOR_B64:
4866 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4867 Inst.eraseFromParent();
4870 case AMDGPU::S_NAND_B64:
4871 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4872 Inst.eraseFromParent();
4875 case AMDGPU::S_NOR_B64:
4876 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4877 Inst.eraseFromParent();
4880 case AMDGPU::S_XNOR_B64:
4881 if (ST.hasDLInsts())
4882 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4884 splitScalar64BitXnor(Worklist, Inst, MDT);
4885 Inst.eraseFromParent();
4888 case AMDGPU::S_ANDN2_B64:
4889 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4890 Inst.eraseFromParent();
4893 case AMDGPU::S_ORN2_B64:
4894 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4895 Inst.eraseFromParent();
4898 case AMDGPU::S_NOT_B64:
4899 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4900 Inst.eraseFromParent();
4903 case AMDGPU::S_BCNT1_I32_B64:
4904 splitScalar64BitBCNT(Worklist, Inst);
4905 Inst.eraseFromParent();
4908 case AMDGPU::S_BFE_I64:
4909 splitScalar64BitBFE(Worklist, Inst);
4910 Inst.eraseFromParent();
4913 case AMDGPU::S_LSHL_B32:
4914 if (ST.hasOnlyRevVALUShifts()) {
4915 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4919 case AMDGPU::S_ASHR_I32:
4920 if (ST.hasOnlyRevVALUShifts()) {
4921 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4925 case AMDGPU::S_LSHR_B32:
4926 if (ST.hasOnlyRevVALUShifts()) {
4927 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4931 case AMDGPU::S_LSHL_B64:
4932 if (ST.hasOnlyRevVALUShifts()) {
4933 NewOpcode = AMDGPU::V_LSHLREV_B64;
4937 case AMDGPU::S_ASHR_I64:
4938 if (ST.hasOnlyRevVALUShifts()) {
4939 NewOpcode = AMDGPU::V_ASHRREV_I64;
4943 case AMDGPU::S_LSHR_B64:
4944 if (ST.hasOnlyRevVALUShifts()) {
4945 NewOpcode = AMDGPU::V_LSHRREV_B64;
4950 case AMDGPU::S_ABS_I32:
4951 lowerScalarAbs(Worklist, Inst);
4952 Inst.eraseFromParent();
4955 case AMDGPU::S_CBRANCH_SCC0:
4956 case AMDGPU::S_CBRANCH_SCC1:
4957 // Clear unused bits of vcc
4959 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4961 .addReg(AMDGPU::EXEC_LO)
4962 .addReg(AMDGPU::VCC_LO);
4964 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4966 .addReg(AMDGPU::EXEC)
4967 .addReg(AMDGPU::VCC);
4970 case AMDGPU::S_BFE_U64:
4971 case AMDGPU::S_BFM_B64:
4972 llvm_unreachable("Moving this op to VALU not implemented");
4974 case AMDGPU::S_PACK_LL_B32_B16:
4975 case AMDGPU::S_PACK_LH_B32_B16:
4976 case AMDGPU::S_PACK_HH_B32_B16:
4977 movePackToVALU(Worklist, MRI, Inst);
4978 Inst.eraseFromParent();
4981 case AMDGPU::S_XNOR_B32:
4982 lowerScalarXnor(Worklist, Inst);
4983 Inst.eraseFromParent();
4986 case AMDGPU::S_NAND_B32:
4987 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4988 Inst.eraseFromParent();
4991 case AMDGPU::S_NOR_B32:
4992 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4993 Inst.eraseFromParent();
4996 case AMDGPU::S_ANDN2_B32:
4997 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4998 Inst.eraseFromParent();
5001 case AMDGPU::S_ORN2_B32:
5002 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
5003 Inst.eraseFromParent();
5007 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
5008 // We cannot move this instruction to the VALU, so we should try to
5009 // legalize its operands instead.
5010 legalizeOperands(Inst, MDT);
5014 // Use the new VALU Opcode.
5015 const MCInstrDesc &NewDesc = get(NewOpcode);
5016 Inst.setDesc(NewDesc);
5018 // Remove any references to SCC. Vector instructions can't read from it, and
5019 // We're just about to add the implicit use / defs of VCC, and we don't want
5021 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
5022 MachineOperand &Op = Inst.getOperand(i);
5023 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
5024 // Only propagate through live-def of SCC.
5025 if (Op.isDef() && !Op.isDead())
5026 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
5027 Inst.RemoveOperand(i);
5031 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
5032 // We are converting these to a BFE, so we need to add the missing
5033 // operands for the size and offset.
5034 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
5035 Inst.addOperand(MachineOperand::CreateImm(0));
5036 Inst.addOperand(MachineOperand::CreateImm(Size));
5038 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
5039 // The VALU version adds the second operand to the result, so insert an
5041 Inst.addOperand(MachineOperand::CreateImm(0));
5044 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent());
5045 fixImplicitOperands(Inst);
5047 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
5048 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
5049 // If we need to move this to VGPRs, we need to unpack the second operand
5050 // back into the 2 separate ones for bit offset and width.
5051 assert(OffsetWidthOp.isImm() &&
5052 "Scalar BFE is only implemented for constant width and offset");
5053 uint32_t Imm = OffsetWidthOp.getImm();
5055 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
5056 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
5057 Inst.RemoveOperand(2); // Remove old immediate.
5058 Inst.addOperand(MachineOperand::CreateImm(Offset));
5059 Inst.addOperand(MachineOperand::CreateImm(BitWidth));
5062 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
5063 unsigned NewDstReg = AMDGPU::NoRegister;
5065 Register DstReg = Inst.getOperand(0).getReg();
5066 if (Register::isPhysicalRegister(DstReg))
5069 // Update the destination register class.
5070 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
5074 if (Inst.isCopy() &&
5075 Register::isVirtualRegister(Inst.getOperand(1).getReg()) &&
5076 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
5077 // Instead of creating a copy where src and dst are the same register
5078 // class, we just replace all uses of dst with src. These kinds of
5079 // copies interfere with the heuristics MachineSink uses to decide
5080 // whether or not to split a critical edge. Since the pass assumes
5081 // that copies will end up as machine instructions and not be
5083 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
5084 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
5085 MRI.clearKillFlags(Inst.getOperand(1).getReg());
5086 Inst.getOperand(0).setReg(DstReg);
5088 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
5089 // these are deleted later, but at -O0 it would leave a suspicious
5090 // looking illegal copy of an undef register.
5091 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
5092 Inst.RemoveOperand(I);
5093 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
5097 NewDstReg = MRI.createVirtualRegister(NewDstRC);
5098 MRI.replaceRegWith(DstReg, NewDstReg);
5101 // Legalize the operands
5102 legalizeOperands(Inst, MDT);
5105 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
5109 // Add/sub require special handling to deal with carry outs.
5110 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
5111 MachineDominatorTree *MDT) const {
5112 if (ST.hasAddNoCarry()) {
5113 // Assume there is no user of scc since we don't select this in that case.
5114 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
5117 MachineBasicBlock &MBB = *Inst.getParent();
5118 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5120 Register OldDstReg = Inst.getOperand(0).getReg();
5121 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5123 unsigned Opc = Inst.getOpcode();
5124 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
5126 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
5127 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
5129 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5130 Inst.RemoveOperand(3);
5132 Inst.setDesc(get(NewOpc));
5133 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5134 Inst.addImplicitDefUseOperands(*MBB.getParent());
5135 MRI.replaceRegWith(OldDstReg, ResultReg);
5136 legalizeOperands(Inst, MDT);
5138 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5145 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
5146 MachineInstr &Inst) const {
5147 MachineBasicBlock &MBB = *Inst.getParent();
5148 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5149 MachineBasicBlock::iterator MII = Inst;
5150 DebugLoc DL = Inst.getDebugLoc();
5152 MachineOperand &Dest = Inst.getOperand(0);
5153 MachineOperand &Src = Inst.getOperand(1);
5154 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5155 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5157 unsigned SubOp = ST.hasAddNoCarry() ?
5158 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
5160 BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
5162 .addReg(Src.getReg());
5164 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
5165 .addReg(Src.getReg())
5168 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5169 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5172 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
5173 MachineInstr &Inst) const {
5174 MachineBasicBlock &MBB = *Inst.getParent();
5175 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5176 MachineBasicBlock::iterator MII = Inst;
5177 const DebugLoc &DL = Inst.getDebugLoc();
5179 MachineOperand &Dest = Inst.getOperand(0);
5180 MachineOperand &Src0 = Inst.getOperand(1);
5181 MachineOperand &Src1 = Inst.getOperand(2);
5183 if (ST.hasDLInsts()) {
5184 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5185 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
5186 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
5188 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
5192 MRI.replaceRegWith(Dest.getReg(), NewDest);
5193 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5195 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5196 // invert either source and then perform the XOR. If either source is a
5197 // scalar register, then we can leave the inversion on the scalar unit to
5198 // acheive a better distrubution of scalar and vector instructions.
5199 bool Src0IsSGPR = Src0.isReg() &&
5200 RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
5201 bool Src1IsSGPR = Src1.isReg() &&
5202 RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
5204 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5205 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5207 // Build a pair of scalar instructions and add them to the work list.
5208 // The next iteration over the work list will lower these to the vector
5209 // unit as necessary.
5211 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
5212 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5215 } else if (Src1IsSGPR) {
5216 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
5217 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5221 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
5225 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
5226 Worklist.insert(Not);
5229 MRI.replaceRegWith(Dest.getReg(), NewDest);
5231 Worklist.insert(Xor);
5233 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5237 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
5239 unsigned Opcode) const {
5240 MachineBasicBlock &MBB = *Inst.getParent();
5241 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5242 MachineBasicBlock::iterator MII = Inst;
5243 const DebugLoc &DL = Inst.getDebugLoc();
5245 MachineOperand &Dest = Inst.getOperand(0);
5246 MachineOperand &Src0 = Inst.getOperand(1);
5247 MachineOperand &Src1 = Inst.getOperand(2);
5249 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5250 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5252 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
5256 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
5259 Worklist.insert(&Op);
5260 Worklist.insert(&Not);
5262 MRI.replaceRegWith(Dest.getReg(), NewDest);
5263 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5266 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
5268 unsigned Opcode) const {
5269 MachineBasicBlock &MBB = *Inst.getParent();
5270 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5271 MachineBasicBlock::iterator MII = Inst;
5272 const DebugLoc &DL = Inst.getDebugLoc();
5274 MachineOperand &Dest = Inst.getOperand(0);
5275 MachineOperand &Src0 = Inst.getOperand(1);
5276 MachineOperand &Src1 = Inst.getOperand(2);
5278 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5279 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5281 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
5284 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
5288 Worklist.insert(&Not);
5289 Worklist.insert(&Op);
5291 MRI.replaceRegWith(Dest.getReg(), NewDest);
5292 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5295 void SIInstrInfo::splitScalar64BitUnaryOp(
5296 SetVectorType &Worklist, MachineInstr &Inst,
5297 unsigned Opcode) const {
5298 MachineBasicBlock &MBB = *Inst.getParent();
5299 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5301 MachineOperand &Dest = Inst.getOperand(0);
5302 MachineOperand &Src0 = Inst.getOperand(1);
5303 DebugLoc DL = Inst.getDebugLoc();
5305 MachineBasicBlock::iterator MII = Inst;
5307 const MCInstrDesc &InstDesc = get(Opcode);
5308 const TargetRegisterClass *Src0RC = Src0.isReg() ?
5309 MRI.getRegClass(Src0.getReg()) :
5310 &AMDGPU::SGPR_32RegClass;
5312 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5314 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5315 AMDGPU::sub0, Src0SubRC);
5317 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5318 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5319 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5321 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5322 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
5324 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5325 AMDGPU::sub1, Src0SubRC);
5327 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5328 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
5330 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
5331 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5333 .addImm(AMDGPU::sub0)
5335 .addImm(AMDGPU::sub1);
5337 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5339 Worklist.insert(&LoHalf);
5340 Worklist.insert(&HiHalf);
5342 // We don't need to legalizeOperands here because for a single operand, src0
5343 // will support any kind of input.
5345 // Move all users of this moved value.
5346 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5349 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
5351 MachineDominatorTree *MDT) const {
5352 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5354 MachineBasicBlock &MBB = *Inst.getParent();
5355 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5356 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5358 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5359 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5360 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5362 Register CarryReg = MRI.createVirtualRegister(CarryRC);
5363 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
5365 MachineOperand &Dest = Inst.getOperand(0);
5366 MachineOperand &Src0 = Inst.getOperand(1);
5367 MachineOperand &Src1 = Inst.getOperand(2);
5368 const DebugLoc &DL = Inst.getDebugLoc();
5369 MachineBasicBlock::iterator MII = Inst;
5371 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
5372 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
5373 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5374 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5376 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5377 AMDGPU::sub0, Src0SubRC);
5378 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5379 AMDGPU::sub0, Src1SubRC);
5382 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5383 AMDGPU::sub1, Src0SubRC);
5384 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5385 AMDGPU::sub1, Src1SubRC);
5387 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
5388 MachineInstr *LoHalf =
5389 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
5390 .addReg(CarryReg, RegState::Define)
5393 .addImm(0); // clamp bit
5395 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
5396 MachineInstr *HiHalf =
5397 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
5398 .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
5401 .addReg(CarryReg, RegState::Kill)
5402 .addImm(0); // clamp bit
5404 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5406 .addImm(AMDGPU::sub0)
5408 .addImm(AMDGPU::sub1);
5410 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5412 // Try to legalize the operands in case we need to swap the order to keep it
5414 legalizeOperands(*LoHalf, MDT);
5415 legalizeOperands(*HiHalf, MDT);
5417 // Move all users of this moved vlaue.
5418 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5421 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
5422 MachineInstr &Inst, unsigned Opcode,
5423 MachineDominatorTree *MDT) const {
5424 MachineBasicBlock &MBB = *Inst.getParent();
5425 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5427 MachineOperand &Dest = Inst.getOperand(0);
5428 MachineOperand &Src0 = Inst.getOperand(1);
5429 MachineOperand &Src1 = Inst.getOperand(2);
5430 DebugLoc DL = Inst.getDebugLoc();
5432 MachineBasicBlock::iterator MII = Inst;
5434 const MCInstrDesc &InstDesc = get(Opcode);
5435 const TargetRegisterClass *Src0RC = Src0.isReg() ?
5436 MRI.getRegClass(Src0.getReg()) :
5437 &AMDGPU::SGPR_32RegClass;
5439 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5440 const TargetRegisterClass *Src1RC = Src1.isReg() ?
5441 MRI.getRegClass(Src1.getReg()) :
5442 &AMDGPU::SGPR_32RegClass;
5444 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5446 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5447 AMDGPU::sub0, Src0SubRC);
5448 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5449 AMDGPU::sub0, Src1SubRC);
5450 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5451 AMDGPU::sub1, Src0SubRC);
5452 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5453 AMDGPU::sub1, Src1SubRC);
5455 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5456 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5457 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5459 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5460 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
5464 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5465 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
5469 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
5470 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5472 .addImm(AMDGPU::sub0)
5474 .addImm(AMDGPU::sub1);
5476 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5478 Worklist.insert(&LoHalf);
5479 Worklist.insert(&HiHalf);
5481 // Move all users of this moved vlaue.
5482 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5485 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist,
5487 MachineDominatorTree *MDT) const {
5488 MachineBasicBlock &MBB = *Inst.getParent();
5489 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5491 MachineOperand &Dest = Inst.getOperand(0);
5492 MachineOperand &Src0 = Inst.getOperand(1);
5493 MachineOperand &Src1 = Inst.getOperand(2);
5494 const DebugLoc &DL = Inst.getDebugLoc();
5496 MachineBasicBlock::iterator MII = Inst;
5498 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5500 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5502 MachineOperand* Op0;
5503 MachineOperand* Op1;
5505 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {
5513 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
5516 Register NewDest = MRI.createVirtualRegister(DestRC);
5518 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
5522 MRI.replaceRegWith(Dest.getReg(), NewDest);
5524 Worklist.insert(&Xor);
5527 void SIInstrInfo::splitScalar64BitBCNT(
5528 SetVectorType &Worklist, MachineInstr &Inst) const {
5529 MachineBasicBlock &MBB = *Inst.getParent();
5530 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5532 MachineBasicBlock::iterator MII = Inst;
5533 const DebugLoc &DL = Inst.getDebugLoc();
5535 MachineOperand &Dest = Inst.getOperand(0);
5536 MachineOperand &Src = Inst.getOperand(1);
5538 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
5539 const TargetRegisterClass *SrcRC = Src.isReg() ?
5540 MRI.getRegClass(Src.getReg()) :
5541 &AMDGPU::SGPR_32RegClass;
5543 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5544 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5546 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
5548 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
5549 AMDGPU::sub0, SrcSubRC);
5550 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
5551 AMDGPU::sub1, SrcSubRC);
5553 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
5555 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
5557 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5559 // We don't need to legalize operands here. src0 for etiher instruction can be
5560 // an SGPR, and the second input is unused or determined here.
5561 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5564 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
5565 MachineInstr &Inst) const {
5566 MachineBasicBlock &MBB = *Inst.getParent();
5567 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5568 MachineBasicBlock::iterator MII = Inst;
5569 const DebugLoc &DL = Inst.getDebugLoc();
5571 MachineOperand &Dest = Inst.getOperand(0);
5572 uint32_t Imm = Inst.getOperand(2).getImm();
5573 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
5574 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
5578 // Only sext_inreg cases handled.
5579 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
5580 Offset == 0 && "Not implemented");
5582 if (BitWidth < 32) {
5583 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5584 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5585 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5587 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
5588 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
5592 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
5596 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
5598 .addImm(AMDGPU::sub0)
5600 .addImm(AMDGPU::sub1);
5602 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5603 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5607 MachineOperand &Src = Inst.getOperand(1);
5608 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5609 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5611 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
5613 .addReg(Src.getReg(), 0, AMDGPU::sub0);
5615 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
5616 .addReg(Src.getReg(), 0, AMDGPU::sub0)
5617 .addImm(AMDGPU::sub0)
5619 .addImm(AMDGPU::sub1);
5621 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5622 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5625 void SIInstrInfo::addUsersToMoveToVALUWorklist(
5627 MachineRegisterInfo &MRI,
5628 SetVectorType &Worklist) const {
5629 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
5630 E = MRI.use_end(); I != E;) {
5631 MachineInstr &UseMI = *I->getParent();
5635 switch (UseMI.getOpcode()) {
5638 case AMDGPU::SOFT_WQM:
5640 case AMDGPU::REG_SEQUENCE:
5642 case AMDGPU::INSERT_SUBREG:
5645 OpNo = I.getOperandNo();
5649 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) {
5650 Worklist.insert(&UseMI);
5654 } while (I != E && I->getParent() == &UseMI);
5661 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
5662 MachineRegisterInfo &MRI,
5663 MachineInstr &Inst) const {
5664 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5665 MachineBasicBlock *MBB = Inst.getParent();
5666 MachineOperand &Src0 = Inst.getOperand(1);
5667 MachineOperand &Src1 = Inst.getOperand(2);
5668 const DebugLoc &DL = Inst.getDebugLoc();
5670 switch (Inst.getOpcode()) {
5671 case AMDGPU::S_PACK_LL_B32_B16: {
5672 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5673 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5675 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
5677 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
5680 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
5681 .addReg(ImmReg, RegState::Kill)
5684 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg)
5687 .addReg(TmpReg, RegState::Kill);
5690 case AMDGPU::S_PACK_LH_B32_B16: {
5691 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5692 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
5694 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg)
5695 .addReg(ImmReg, RegState::Kill)
5700 case AMDGPU::S_PACK_HH_B32_B16: {
5701 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5702 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5703 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
5706 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
5707 .addImm(0xffff0000);
5708 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg)
5710 .addReg(ImmReg, RegState::Kill)
5711 .addReg(TmpReg, RegState::Kill);
5715 llvm_unreachable("unhandled s_pack_* instruction");
5718 MachineOperand &Dest = Inst.getOperand(0);
5719 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5720 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5723 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
5724 MachineInstr &SCCDefInst,
5725 SetVectorType &Worklist) const {
5726 // Ensure that def inst defines SCC, which is still live.
5727 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
5728 !Op.isDead() && Op.getParent() == &SCCDefInst);
5729 // This assumes that all the users of SCC are in the same block
5731 for (MachineInstr &MI : // Skip the def inst itself.
5732 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)),
5733 SCCDefInst.getParent()->end())) {
5734 // Check if SCC is used first.
5735 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1)
5736 Worklist.insert(&MI);
5737 // Exit if we find another SCC def.
5738 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
5743 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
5744 const MachineInstr &Inst) const {
5745 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
5747 switch (Inst.getOpcode()) {
5748 // For target instructions, getOpRegClass just returns the virtual register
5749 // class associated with the operand, so we need to find an equivalent VGPR
5750 // register class in order to move the instruction to the VALU.
5753 case AMDGPU::REG_SEQUENCE:
5754 case AMDGPU::INSERT_SUBREG:
5756 case AMDGPU::SOFT_WQM:
5758 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1);
5759 if (RI.hasAGPRs(SrcRC)) {
5760 if (RI.hasAGPRs(NewDstRC))
5763 switch (Inst.getOpcode()) {
5765 case AMDGPU::REG_SEQUENCE:
5766 case AMDGPU::INSERT_SUBREG:
5767 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
5770 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
5776 if (RI.hasVGPRs(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
5779 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
5791 // Find the one SGPR operand we are allowed to use.
5792 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
5793 int OpIndices[3]) const {
5794 const MCInstrDesc &Desc = MI.getDesc();
5796 // Find the one SGPR operand we are allowed to use.
5798 // First we need to consider the instruction's operand requirements before
5799 // legalizing. Some operands are required to be SGPRs, such as implicit uses
5800 // of VCC, but we are still bound by the constant bus requirement to only use
5803 // If the operand's class is an SGPR, we can never move it.
5805 unsigned SGPRReg = findImplicitSGPRRead(MI);
5806 if (SGPRReg != AMDGPU::NoRegister)
5809 unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
5810 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5812 for (unsigned i = 0; i < 3; ++i) {
5813 int Idx = OpIndices[i];
5817 const MachineOperand &MO = MI.getOperand(Idx);
5821 // Is this operand statically required to be an SGPR based on the operand
5823 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
5824 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
5828 // If this could be a VGPR or an SGPR, Check the dynamic register class.
5829 Register Reg = MO.getReg();
5830 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
5831 if (RI.isSGPRClass(RegRC))
5835 // We don't have a required SGPR operand, so we have a bit more freedom in
5836 // selecting operands to move.
5838 // Try to select the most used SGPR. If an SGPR is equal to one of the
5839 // others, we choose that.
5842 // V_FMA_F32 v0, s0, s0, s0 -> No moves
5843 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
5845 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
5848 if (UsedSGPRs[0] != AMDGPU::NoRegister) {
5849 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
5850 SGPRReg = UsedSGPRs[0];
5853 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
5854 if (UsedSGPRs[1] == UsedSGPRs[2])
5855 SGPRReg = UsedSGPRs[1];
5861 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
5862 unsigned OperandName) const {
5863 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
5867 return &MI.getOperand(Idx);
5870 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
5871 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
5872 return (22ULL << 44) | // IMG_FORMAT_32_FLOAT
5873 (1ULL << 56) | // RESOURCE_LEVEL = 1
5874 (3ULL << 60); // OOB_SELECT = 3
5877 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
5878 if (ST.isAmdHsaOS()) {
5879 // Set ATC = 1. GFX9 doesn't have this bit.
5880 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
5881 RsrcDataFormat |= (1ULL << 56);
5883 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
5884 // BTW, it disables TC L2 and therefore decreases performance.
5885 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
5886 RsrcDataFormat |= (2ULL << 59);
5889 return RsrcDataFormat;
5892 uint64_t SIInstrInfo::getScratchRsrcWords23() const {
5893 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
5894 AMDGPU::RSRC_TID_ENABLE |
5895 0xffffffff; // Size;
5897 // GFX9 doesn't have ELEMENT_SIZE.
5898 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5899 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
5900 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
5903 // IndexStride = 64 / 32.
5904 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2;
5905 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
5907 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
5908 // Clear them unless we want a huge stride.
5909 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
5910 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
5911 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
5916 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
5917 unsigned Opc = MI.getOpcode();
5922 bool SIInstrInfo::isHighLatencyInstruction(const MachineInstr &MI) const {
5923 unsigned Opc = MI.getOpcode();
5925 return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
5928 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
5929 int &FrameIndex) const {
5930 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
5931 if (!Addr || !Addr->isFI())
5932 return AMDGPU::NoRegister;
5934 assert(!MI.memoperands_empty() &&
5935 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
5937 FrameIndex = Addr->getIndex();
5938 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
5941 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
5942 int &FrameIndex) const {
5943 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
5944 assert(Addr && Addr->isFI());
5945 FrameIndex = Addr->getIndex();
5946 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
5949 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
5950 int &FrameIndex) const {
5952 return AMDGPU::NoRegister;
5954 if (isMUBUF(MI) || isVGPRSpill(MI))
5955 return isStackAccess(MI, FrameIndex);
5957 if (isSGPRSpill(MI))
5958 return isSGPRStackAccess(MI, FrameIndex);
5960 return AMDGPU::NoRegister;
5963 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
5964 int &FrameIndex) const {
5966 return AMDGPU::NoRegister;
5968 if (isMUBUF(MI) || isVGPRSpill(MI))
5969 return isStackAccess(MI, FrameIndex);
5971 if (isSGPRSpill(MI))
5972 return isSGPRStackAccess(MI, FrameIndex);
5974 return AMDGPU::NoRegister;
5977 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const {
5979 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
5980 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
5981 while (++I != E && I->isInsideBundle()) {
5982 assert(!I->isBundle() && "No nested bundle!");
5983 Size += getInstSizeInBytes(*I);
5989 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
5990 unsigned Opc = MI.getOpcode();
5991 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
5992 unsigned DescSize = Desc.getSize();
5994 // If we have a definitive size, we can use it. Otherwise we need to inspect
5995 // the operands to know the size.
5996 if (isFixedSize(MI))
5999 // 4-byte instructions may have a 32-bit literal encoded after them. Check
6000 // operands that coud ever be literals.
6001 if (isVALU(MI) || isSALU(MI)) {
6002 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
6004 return DescSize; // No operands.
6006 if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx]))
6007 return isVOP3(MI) ? 12 : (DescSize + 4);
6009 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
6013 if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx]))
6014 return isVOP3(MI) ? 12 : (DescSize + 4);
6016 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
6020 if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx]))
6021 return isVOP3(MI) ? 12 : (DescSize + 4);
6026 // Check whether we have extra NSA words.
6028 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
6032 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
6033 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
6037 case TargetOpcode::IMPLICIT_DEF:
6038 case TargetOpcode::KILL:
6039 case TargetOpcode::DBG_VALUE:
6040 case TargetOpcode::EH_LABEL:
6042 case TargetOpcode::BUNDLE:
6043 return getInstBundleSize(MI);
6044 case TargetOpcode::INLINEASM:
6045 case TargetOpcode::INLINEASM_BR: {
6046 const MachineFunction *MF = MI.getParent()->getParent();
6047 const char *AsmStr = MI.getOperand(0).getSymbolName();
6048 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(),
6049 &MF->getSubtarget());
6056 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
6060 if (MI.memoperands_empty())
6063 for (const MachineMemOperand *MMO : MI.memoperands()) {
6064 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
6070 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const {
6071 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
6074 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
6075 MachineBasicBlock *IfEnd) const {
6076 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator();
6077 assert(TI != IfEntry->end());
6079 MachineInstr *Branch = &(*TI);
6080 MachineFunction *MF = IfEntry->getParent();
6081 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
6083 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
6084 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC());
6085 MachineInstr *SIIF =
6086 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
6087 .add(Branch->getOperand(0))
6088 .add(Branch->getOperand(1));
6089 MachineInstr *SIEND =
6090 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
6094 IfEntry->insert(IfEntry->end(), SIIF);
6095 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND);
6099 void SIInstrInfo::convertNonUniformLoopRegion(
6100 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
6101 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator();
6102 // We expect 2 terminators, one conditional and one unconditional.
6103 assert(TI != LoopEnd->end());
6105 MachineInstr *Branch = &(*TI);
6106 MachineFunction *MF = LoopEnd->getParent();
6107 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
6109 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
6111 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC());
6112 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC());
6113 MachineInstrBuilder HeaderPHIBuilder =
6114 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg);
6115 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(),
6116 E = LoopEntry->pred_end();
6118 if (*PI == LoopEnd) {
6119 HeaderPHIBuilder.addReg(BackEdgeReg);
6121 MachineBasicBlock *PMBB = *PI;
6122 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC());
6123 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(),
6125 HeaderPHIBuilder.addReg(ZeroReg);
6127 HeaderPHIBuilder.addMBB(*PI);
6129 MachineInstr *HeaderPhi = HeaderPHIBuilder;
6130 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(),
6131 get(AMDGPU::SI_IF_BREAK), BackEdgeReg)
6133 .add(Branch->getOperand(0));
6134 MachineInstr *SILOOP =
6135 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP))
6136 .addReg(BackEdgeReg)
6139 LoopEntry->insert(LoopEntry->begin(), HeaderPhi);
6141 LoopEnd->insert(LoopEnd->end(), SIIFBREAK);
6142 LoopEnd->insert(LoopEnd->end(), SILOOP);
6146 ArrayRef<std::pair<int, const char *>>
6147 SIInstrInfo::getSerializableTargetIndices() const {
6148 static const std::pair<int, const char *> TargetIndices[] = {
6149 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
6150 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
6151 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
6152 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
6153 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
6154 return makeArrayRef(TargetIndices);
6157 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
6158 /// post-RA version of misched uses CreateTargetMIHazardRecognizer.
6159 ScheduleHazardRecognizer *
6160 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
6161 const ScheduleDAG *DAG) const {
6162 return new GCNHazardRecognizer(DAG->MF);
6165 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
6167 ScheduleHazardRecognizer *
6168 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
6169 return new GCNHazardRecognizer(MF);
6172 std::pair<unsigned, unsigned>
6173 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
6174 return std::make_pair(TF & MO_MASK, TF & ~MO_MASK);
6177 ArrayRef<std::pair<unsigned, const char *>>
6178 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
6179 static const std::pair<unsigned, const char *> TargetFlags[] = {
6180 { MO_GOTPCREL, "amdgpu-gotprel" },
6181 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" },
6182 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" },
6183 { MO_REL32_LO, "amdgpu-rel32-lo" },
6184 { MO_REL32_HI, "amdgpu-rel32-hi" },
6185 { MO_ABS32_LO, "amdgpu-abs32-lo" },
6186 { MO_ABS32_HI, "amdgpu-abs32-hi" },
6189 return makeArrayRef(TargetFlags);
6192 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const {
6193 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
6194 MI.modifiesRegister(AMDGPU::EXEC, &RI);
6198 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
6199 MachineBasicBlock::iterator I,
6201 unsigned DestReg) const {
6202 if (ST.hasAddNoCarry())
6203 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
6205 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
6206 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC());
6207 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC());
6209 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
6210 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
6213 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
6214 MachineBasicBlock::iterator I,
6217 RegScavenger &RS) const {
6218 if (ST.hasAddNoCarry())
6219 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg);
6221 // If available, prefer to use vcc.
6222 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC)
6223 ? Register(RI.getVCC())
6224 : RS.scavengeRegister(RI.getBoolRC(), I, 0, false);
6226 // TODO: Users need to deal with this.
6227 if (!UnusedCarry.isValid())
6228 return MachineInstrBuilder();
6230 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
6231 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
6234 bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
6236 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
6237 case AMDGPU::SI_KILL_I1_TERMINATOR:
6244 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
6246 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
6247 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
6248 case AMDGPU::SI_KILL_I1_PSEUDO:
6249 return get(AMDGPU::SI_KILL_I1_TERMINATOR);
6251 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
6255 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
6256 MachineBasicBlock *MBB = MI.getParent();
6257 MachineFunction *MF = MBB->getParent();
6258 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6263 for (auto &Op : MI.implicit_operands()) {
6264 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
6265 Op.setReg(AMDGPU::VCC_LO);
6269 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
6273 // Check that it is using a buffer resource.
6274 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
6275 if (Idx == -1) // e.g. s_memtime
6278 const auto RCID = MI.getDesc().OpInfo[Idx].RegClass;
6279 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass);
6282 unsigned SIInstrInfo::getNumFlatOffsetBits(unsigned AddrSpace,
6283 bool Signed) const {
6284 if (!ST.hasFlatInstOffsets())
6287 if (ST.hasFlatSegmentOffsetBug() && AddrSpace == AMDGPUAS::FLAT_ADDRESS)
6290 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10)
6291 return Signed ? 12 : 11;
6293 return Signed ? 13 : 12;
6296 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
6297 bool Signed) const {
6298 // TODO: Should 0 be special cased?
6299 if (!ST.hasFlatInstOffsets())
6302 if (ST.hasFlatSegmentOffsetBug() && AddrSpace == AMDGPUAS::FLAT_ADDRESS)
6305 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
6306 return (Signed && isInt<12>(Offset)) ||
6307 (!Signed && isUInt<11>(Offset));
6310 return (Signed && isInt<13>(Offset)) ||
6311 (!Signed && isUInt<12>(Offset));
6315 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td
6316 enum SIEncodingFamily {
6327 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) {
6328 switch (ST.getGeneration()) {
6331 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
6332 case AMDGPUSubtarget::SEA_ISLANDS:
6333 return SIEncodingFamily::SI;
6334 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
6335 case AMDGPUSubtarget::GFX9:
6336 return SIEncodingFamily::VI;
6337 case AMDGPUSubtarget::GFX10:
6338 return SIEncodingFamily::GFX10;
6340 llvm_unreachable("Unknown subtarget generation!");
6343 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
6345 // These opcodes use indirect register addressing so
6346 // they need special handling by codegen (currently missing).
6347 // Therefore it is too risky to allow these opcodes
6348 // to be selected by dpp combiner or sdwa peepholer.
6349 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
6350 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
6351 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
6352 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
6353 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
6354 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
6355 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
6356 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
6363 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
6364 SIEncodingFamily Gen = subtargetEncodingFamily(ST);
6366 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
6367 ST.getGeneration() == AMDGPUSubtarget::GFX9)
6368 Gen = SIEncodingFamily::GFX9;
6370 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
6371 // subtarget has UnpackedD16VMem feature.
6372 // TODO: remove this when we discard GFX80 encoding.
6373 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))
6374 Gen = SIEncodingFamily::GFX80;
6376 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) {
6377 switch (ST.getGeneration()) {
6379 Gen = SIEncodingFamily::SDWA;
6381 case AMDGPUSubtarget::GFX9:
6382 Gen = SIEncodingFamily::SDWA9;
6384 case AMDGPUSubtarget::GFX10:
6385 Gen = SIEncodingFamily::SDWA10;
6390 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
6392 // -1 means that Opcode is already a native instruction.
6396 // (uint16_t)-1 means that Opcode is a pseudo instruction that has
6397 // no encoding in the given subtarget generation.
6398 if (MCOp == (uint16_t)-1)
6401 if (isAsmOnlyOpcode(MCOp))
6408 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
6409 assert(RegOpnd.isReg());
6410 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
6411 getRegSubRegPair(RegOpnd);
6414 TargetInstrInfo::RegSubRegPair
6415 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
6416 assert(MI.isRegSequence());
6417 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
6418 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) {
6419 auto &RegOp = MI.getOperand(1 + 2 * I);
6420 return getRegOrUndef(RegOp);
6422 return TargetInstrInfo::RegSubRegPair();
6425 // Try to find the definition of reg:subreg in subreg-manipulation pseudos
6426 // Following a subreg of reg:subreg isn't supported
6427 static bool followSubRegDef(MachineInstr &MI,
6428 TargetInstrInfo::RegSubRegPair &RSR) {
6431 switch (MI.getOpcode()) {
6433 case AMDGPU::REG_SEQUENCE:
6434 RSR = getRegSequenceSubReg(MI, RSR.SubReg);
6436 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
6437 case AMDGPU::INSERT_SUBREG:
6438 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm())
6439 // inserted the subreg we're looking for
6440 RSR = getRegOrUndef(MI.getOperand(2));
6441 else { // the subreg in the rest of the reg
6442 auto R1 = getRegOrUndef(MI.getOperand(1));
6443 if (R1.SubReg) // subreg of subreg isn't supported
6452 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
6453 MachineRegisterInfo &MRI) {
6454 assert(MRI.isSSA());
6455 if (!Register::isVirtualRegister(P.Reg))
6459 auto *DefInst = MRI.getVRegDef(RSR.Reg);
6460 while (auto *MI = DefInst) {
6462 switch (MI->getOpcode()) {
6464 case AMDGPU::V_MOV_B32_e32: {
6465 auto &Op1 = MI->getOperand(1);
6466 if (Op1.isReg() && Register::isVirtualRegister(Op1.getReg())) {
6469 RSR = getRegSubRegPair(Op1);
6470 DefInst = MRI.getVRegDef(RSR.Reg);
6475 if (followSubRegDef(*MI, RSR)) {
6478 DefInst = MRI.getVRegDef(RSR.Reg);
6487 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
6489 const MachineInstr &DefMI,
6490 const MachineInstr &UseMI) {
6491 assert(MRI.isSSA() && "Must be run on SSA");
6493 auto *TRI = MRI.getTargetRegisterInfo();
6494 auto *DefBB = DefMI.getParent();
6496 // Don't bother searching between blocks, although it is possible this block
6497 // doesn't modify exec.
6498 if (UseMI.getParent() != DefBB)
6501 const int MaxInstScan = 20;
6504 // Stop scan at the use.
6505 auto E = UseMI.getIterator();
6506 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) {
6507 if (I->isDebugInstr())
6510 if (++NumInst > MaxInstScan)
6513 if (I->modifiesRegister(AMDGPU::EXEC, TRI))
6520 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
6522 const MachineInstr &DefMI) {
6523 assert(MRI.isSSA() && "Must be run on SSA");
6525 auto *TRI = MRI.getTargetRegisterInfo();
6526 auto *DefBB = DefMI.getParent();
6528 const int MaxUseInstScan = 10;
6531 for (auto &UseInst : MRI.use_nodbg_instructions(VReg)) {
6532 // Don't bother searching between blocks, although it is possible this block
6533 // doesn't modify exec.
6534 if (UseInst.getParent() != DefBB)
6537 if (++NumUseInst > MaxUseInstScan)
6541 const int MaxInstScan = 20;
6544 // Stop scan when we have seen all the uses.
6545 for (auto I = std::next(DefMI.getIterator()); ; ++I) {
6546 if (I->isDebugInstr())
6549 if (++NumInst > MaxInstScan)
6552 if (I->readsRegister(VReg))
6553 if (--NumUseInst == 0)
6556 if (I->modifiesRegister(AMDGPU::EXEC, TRI))
6561 MachineInstr *SIInstrInfo::createPHIDestinationCopy(
6562 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
6563 const DebugLoc &DL, Register Src, Register Dst) const {
6564 auto Cur = MBB.begin();
6565 if (Cur != MBB.end())
6567 if (!Cur->isPHI() && Cur->readsRegister(Dst))
6568 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src);
6570 } while (Cur != MBB.end() && Cur != LastPHIIt);
6572 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src,
6576 MachineInstr *SIInstrInfo::createPHISourceCopy(
6577 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
6578 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
6579 if (InsPt != MBB.end() &&
6580 (InsPt->getOpcode() == AMDGPU::SI_IF ||
6581 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
6582 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
6583 InsPt->definesRegister(Src)) {
6585 return BuildMI(MBB, InsPt, DL,
6586 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term
6587 : AMDGPU::S_MOV_B64_term),
6589 .addReg(Src, 0, SrcSubReg)
6590 .addReg(AMDGPU::EXEC, RegState::Implicit);
6592 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
6596 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
6598 MachineInstr *SIInstrInfo::foldMemoryOperandImpl(
6599 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
6600 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
6601 VirtRegMap *VRM) const {
6602 // This is a bit of a hack (copied from AArch64). Consider this instruction:
6604 // %0:sreg_32 = COPY $m0
6606 // We explicitly chose SReg_32 for the virtual register so such a copy might
6607 // be eliminated by RegisterCoalescer. However, that may not be possible, and
6608 // %0 may even spill. We can't spill $m0 normally (it would require copying to
6609 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
6610 // TargetInstrInfo::foldMemoryOperand() is going to try.
6612 // To prevent that, constrain the %0 register class here.
6613 if (MI.isFullCopy()) {
6614 Register DstReg = MI.getOperand(0).getReg();
6615 Register SrcReg = MI.getOperand(1).getReg();
6617 if (DstReg == AMDGPU::M0 && SrcReg.isVirtual()) {
6618 MF.getRegInfo().constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
6622 if (SrcReg == AMDGPU::M0 && DstReg.isVirtual()) {
6623 MF.getRegInfo().constrainRegClass(DstReg, &AMDGPU::SReg_32_XM0RegClass);
6631 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
6632 const MachineInstr &MI,
6633 unsigned *PredCost) const {
6634 if (MI.isBundle()) {
6635 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
6636 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
6637 unsigned Lat = 0, Count = 0;
6638 for (++I; I != E && I->isBundledWithPred(); ++I) {
6640 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I));
6642 return Lat + Count - 1;
6645 return SchedModel.computeInstrLatency(&MI);