1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// SI Implementation of TargetInstrInfo.
13 //===----------------------------------------------------------------------===//
15 #include "SIInstrInfo.h"
17 #include "AMDGPUIntrinsicInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "GCNHazardRecognizer.h"
20 #include "SIDefines.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "SIRegisterInfo.h"
23 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
24 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/ADT/APInt.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
29 #include "llvm/ADT/iterator_range.h"
30 #include "llvm/Analysis/AliasAnalysis.h"
31 #include "llvm/Analysis/MemoryLocation.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/MachineBasicBlock.h"
34 #include "llvm/CodeGen/MachineFrameInfo.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineInstr.h"
37 #include "llvm/CodeGen/MachineInstrBuilder.h"
38 #include "llvm/CodeGen/MachineInstrBundle.h"
39 #include "llvm/CodeGen/MachineMemOperand.h"
40 #include "llvm/CodeGen/MachineOperand.h"
41 #include "llvm/CodeGen/MachineRegisterInfo.h"
42 #include "llvm/CodeGen/RegisterScavenging.h"
43 #include "llvm/CodeGen/ScheduleDAG.h"
44 #include "llvm/CodeGen/SelectionDAGNodes.h"
45 #include "llvm/CodeGen/TargetOpcodes.h"
46 #include "llvm/CodeGen/TargetRegisterInfo.h"
47 #include "llvm/IR/DebugLoc.h"
48 #include "llvm/IR/DiagnosticInfo.h"
49 #include "llvm/IR/Function.h"
50 #include "llvm/IR/InlineAsm.h"
51 #include "llvm/IR/LLVMContext.h"
52 #include "llvm/MC/MCInstrDesc.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/CommandLine.h"
55 #include "llvm/Support/Compiler.h"
56 #include "llvm/Support/ErrorHandling.h"
57 #include "llvm/Support/MachineValueType.h"
58 #include "llvm/Support/MathExtras.h"
59 #include "llvm/Target/TargetMachine.h"
67 #define GET_INSTRINFO_CTOR_DTOR
68 #include "AMDGPUGenInstrInfo.inc"
72 #define GET_D16ImageDimIntrinsics_IMPL
73 #define GET_ImageDimIntrinsicTable_IMPL
74 #define GET_RsrcIntrinsics_IMPL
75 #include "AMDGPUGenSearchableTables.inc"
80 // Must be at least 4 to be able to branch over minimum unconditional branch
81 // code. This is only for making it possible to write reasonably small tests for
83 static cl::opt<unsigned>
84 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
85 cl::desc("Restrict range of branch instructions (DEBUG)"));
87 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
88 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
91 //===----------------------------------------------------------------------===//
92 // TargetInstrInfo callbacks
93 //===----------------------------------------------------------------------===//
95 static unsigned getNumOperandsNoGlue(SDNode *Node) {
96 unsigned N = Node->getNumOperands();
97 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
102 static SDValue findChainOperand(SDNode *Load) {
103 SDValue LastOp = Load->getOperand(getNumOperandsNoGlue(Load) - 1);
104 assert(LastOp.getValueType() == MVT::Other && "Chain missing from load node");
108 /// Returns true if both nodes have the same value for the given
109 /// operand \p Op, or if both nodes do not have this operand.
110 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
111 unsigned Opc0 = N0->getMachineOpcode();
112 unsigned Opc1 = N1->getMachineOpcode();
114 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
115 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
117 if (Op0Idx == -1 && Op1Idx == -1)
121 if ((Op0Idx == -1 && Op1Idx != -1) ||
122 (Op1Idx == -1 && Op0Idx != -1))
125 // getNamedOperandIdx returns the index for the MachineInstr's operands,
126 // which includes the result as the first operand. We are indexing into the
127 // MachineSDNode's operands, so we need to skip the result operand to get
132 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
135 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
136 AliasAnalysis *AA) const {
137 // TODO: The generic check fails for VALU instructions that should be
138 // rematerializable due to implicit reads of exec. We really want all of the
139 // generic logic for this except for this.
140 switch (MI.getOpcode()) {
141 case AMDGPU::V_MOV_B32_e32:
142 case AMDGPU::V_MOV_B32_e64:
143 case AMDGPU::V_MOV_B64_PSEUDO:
150 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
152 int64_t &Offset1) const {
153 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
156 unsigned Opc0 = Load0->getMachineOpcode();
157 unsigned Opc1 = Load1->getMachineOpcode();
159 // Make sure both are actually loads.
160 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
163 if (isDS(Opc0) && isDS(Opc1)) {
165 // FIXME: Handle this case:
166 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
170 if (Load0->getOperand(1) != Load1->getOperand(1))
174 if (findChainOperand(Load0) != findChainOperand(Load1))
177 // Skip read2 / write2 variants for simplicity.
178 // TODO: We should report true if the used offsets are adjacent (excluded
180 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 ||
181 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1)
184 Offset0 = cast<ConstantSDNode>(Load0->getOperand(2))->getZExtValue();
185 Offset1 = cast<ConstantSDNode>(Load1->getOperand(2))->getZExtValue();
189 if (isSMRD(Opc0) && isSMRD(Opc1)) {
190 // Skip time and cache invalidation instructions.
191 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
192 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
195 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1));
198 if (Load0->getOperand(0) != Load1->getOperand(0))
201 const ConstantSDNode *Load0Offset =
202 dyn_cast<ConstantSDNode>(Load0->getOperand(1));
203 const ConstantSDNode *Load1Offset =
204 dyn_cast<ConstantSDNode>(Load1->getOperand(1));
206 if (!Load0Offset || !Load1Offset)
210 if (findChainOperand(Load0) != findChainOperand(Load1))
213 Offset0 = Load0Offset->getZExtValue();
214 Offset1 = Load1Offset->getZExtValue();
218 // MUBUF and MTBUF can access the same addresses.
219 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
221 // MUBUF and MTBUF have vaddr at different indices.
222 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
223 findChainOperand(Load0) != findChainOperand(Load1) ||
224 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
225 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
228 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
229 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
231 if (OffIdx0 == -1 || OffIdx1 == -1)
234 // getNamedOperandIdx returns the index for MachineInstrs. Since they
235 // inlcude the output in the operand list, but SDNodes don't, we need to
236 // subtract the index by one.
240 SDValue Off0 = Load0->getOperand(OffIdx0);
241 SDValue Off1 = Load1->getOperand(OffIdx1);
243 // The offset might be a FrameIndexSDNode.
244 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
247 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
248 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
255 static bool isStride64(unsigned Opc) {
257 case AMDGPU::DS_READ2ST64_B32:
258 case AMDGPU::DS_READ2ST64_B64:
259 case AMDGPU::DS_WRITE2ST64_B32:
260 case AMDGPU::DS_WRITE2ST64_B64:
267 bool SIInstrInfo::getMemOpBaseRegImmOfs(MachineInstr &LdSt, unsigned &BaseReg,
269 const TargetRegisterInfo *TRI) const {
270 unsigned Opc = LdSt.getOpcode();
273 const MachineOperand *OffsetImm =
274 getNamedOperand(LdSt, AMDGPU::OpName::offset);
276 // Normal, single offset LDS instruction.
277 const MachineOperand *AddrReg =
278 getNamedOperand(LdSt, AMDGPU::OpName::addr);
280 BaseReg = AddrReg->getReg();
281 Offset = OffsetImm->getImm();
285 // The 2 offset instructions use offset0 and offset1 instead. We can treat
286 // these as a load with a single offset if the 2 offsets are consecutive. We
287 // will use this for some partially aligned loads.
288 const MachineOperand *Offset0Imm =
289 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
290 const MachineOperand *Offset1Imm =
291 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
293 uint8_t Offset0 = Offset0Imm->getImm();
294 uint8_t Offset1 = Offset1Imm->getImm();
296 if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
297 // Each of these offsets is in element sized units, so we need to convert
298 // to bytes of the individual reads.
302 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
304 assert(LdSt.mayStore());
305 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
306 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
312 const MachineOperand *AddrReg =
313 getNamedOperand(LdSt, AMDGPU::OpName::addr);
314 BaseReg = AddrReg->getReg();
315 Offset = EltSize * Offset0;
322 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
323 const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
324 if (SOffset && SOffset->isReg())
327 const MachineOperand *AddrReg =
328 getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
332 const MachineOperand *OffsetImm =
333 getNamedOperand(LdSt, AMDGPU::OpName::offset);
334 BaseReg = AddrReg->getReg();
335 Offset = OffsetImm->getImm();
337 if (SOffset) // soffset can be an inline immediate.
338 Offset += SOffset->getImm();
344 const MachineOperand *OffsetImm =
345 getNamedOperand(LdSt, AMDGPU::OpName::offset);
349 const MachineOperand *SBaseReg =
350 getNamedOperand(LdSt, AMDGPU::OpName::sbase);
351 BaseReg = SBaseReg->getReg();
352 Offset = OffsetImm->getImm();
357 const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
359 // Can't analyze 2 offsets.
360 if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
363 BaseReg = VAddr->getReg();
365 // scratch instructions have either vaddr or saddr.
366 BaseReg = getNamedOperand(LdSt, AMDGPU::OpName::saddr)->getReg();
369 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
376 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, unsigned BaseReg1,
377 const MachineInstr &MI2, unsigned BaseReg2) {
378 if (BaseReg1 == BaseReg2)
381 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
384 auto MO1 = *MI1.memoperands_begin();
385 auto MO2 = *MI2.memoperands_begin();
386 if (MO1->getAddrSpace() != MO2->getAddrSpace())
389 auto Base1 = MO1->getValue();
390 auto Base2 = MO2->getValue();
391 if (!Base1 || !Base2)
393 const MachineFunction &MF = *MI1.getParent()->getParent();
394 const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
395 Base1 = GetUnderlyingObject(Base1, DL);
396 Base2 = GetUnderlyingObject(Base1, DL);
398 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
401 return Base1 == Base2;
404 bool SIInstrInfo::shouldClusterMemOps(MachineInstr &FirstLdSt,
406 MachineInstr &SecondLdSt,
408 unsigned NumLoads) const {
409 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseReg1, SecondLdSt, BaseReg2))
412 const MachineOperand *FirstDst = nullptr;
413 const MachineOperand *SecondDst = nullptr;
415 if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
416 (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
417 (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
418 const unsigned MaxGlobalLoadCluster = 6;
419 if (NumLoads > MaxGlobalLoadCluster)
422 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
424 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
425 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
427 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
428 } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
429 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
430 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
431 } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
432 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
433 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
436 if (!FirstDst || !SecondDst)
439 // Try to limit clustering based on the total number of bytes loaded
440 // rather than the number of instructions. This is done to help reduce
441 // register pressure. The method used is somewhat inexact, though,
442 // because it assumes that all loads in the cluster will load the
443 // same number of bytes as FirstLdSt.
445 // The unit of this value is bytes.
446 // FIXME: This needs finer tuning.
447 unsigned LoadClusterThreshold = 16;
449 const MachineRegisterInfo &MRI =
450 FirstLdSt.getParent()->getParent()->getRegInfo();
451 const TargetRegisterClass *DstRC = MRI.getRegClass(FirstDst->getReg());
453 return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
456 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
457 // the first 16 loads will be interleaved with the stores, and the next 16 will
458 // be clustered as expected. It should really split into 2 16 store batches.
460 // Loads are clustered until this returns false, rather than trying to schedule
461 // groups of stores. This also means we have to deal with saying different
462 // address space loads should be clustered, and ones which might cause bank
465 // This might be deprecated so it might not be worth that much effort to fix.
466 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
467 int64_t Offset0, int64_t Offset1,
468 unsigned NumLoads) const {
469 assert(Offset1 > Offset0 &&
470 "Second offset should be larger than first offset!");
471 // If we have less than 16 loads in a row, and the offsets are within 64
472 // bytes, then schedule together.
474 // A cacheline is 64 bytes (for global memory).
475 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
478 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
479 MachineBasicBlock::iterator MI,
480 const DebugLoc &DL, unsigned DestReg,
481 unsigned SrcReg, bool KillSrc) {
482 MachineFunction *MF = MBB.getParent();
483 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
484 "illegal SGPR to VGPR copy",
486 LLVMContext &C = MF->getFunction().getContext();
487 C.diagnose(IllegalCopy);
489 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
490 .addReg(SrcReg, getKillRegState(KillSrc));
493 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
494 MachineBasicBlock::iterator MI,
495 const DebugLoc &DL, unsigned DestReg,
496 unsigned SrcReg, bool KillSrc) const {
497 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
499 if (RC == &AMDGPU::VGPR_32RegClass) {
500 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
501 AMDGPU::SReg_32RegClass.contains(SrcReg));
502 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
503 .addReg(SrcReg, getKillRegState(KillSrc));
507 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
508 RC == &AMDGPU::SReg_32RegClass) {
509 if (SrcReg == AMDGPU::SCC) {
510 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
516 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
517 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
521 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
522 .addReg(SrcReg, getKillRegState(KillSrc));
526 if (RC == &AMDGPU::SReg_64RegClass) {
527 if (DestReg == AMDGPU::VCC) {
528 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
529 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
530 .addReg(SrcReg, getKillRegState(KillSrc));
532 // FIXME: Hack until VReg_1 removed.
533 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
534 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
536 .addReg(SrcReg, getKillRegState(KillSrc));
542 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
543 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
547 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
548 .addReg(SrcReg, getKillRegState(KillSrc));
552 if (DestReg == AMDGPU::SCC) {
553 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
554 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
555 .addReg(SrcReg, getKillRegState(KillSrc))
560 unsigned EltSize = 4;
561 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
562 if (RI.isSGPRClass(RC)) {
563 if (RI.getRegSizeInBits(*RC) > 32) {
564 Opcode = AMDGPU::S_MOV_B64;
567 Opcode = AMDGPU::S_MOV_B32;
571 if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
572 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
577 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
578 bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
580 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
583 SubIdx = SubIndices[Idx];
585 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
587 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
588 get(Opcode), RI.getSubReg(DestReg, SubIdx));
590 Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
593 Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
595 bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
596 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
600 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
603 // Try to map original to commuted opcode
604 NewOpc = AMDGPU::getCommuteRev(Opcode);
606 // Check if the commuted (REV) opcode exists on the target.
607 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
609 // Try to map commuted to original opcode
610 NewOpc = AMDGPU::getCommuteOrig(Opcode);
612 // Check if the original (non-REV) opcode exists on the target.
613 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
618 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
619 MachineBasicBlock::iterator MI,
620 const DebugLoc &DL, unsigned DestReg,
621 int64_t Value) const {
622 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
623 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
624 if (RegClass == &AMDGPU::SReg_32RegClass ||
625 RegClass == &AMDGPU::SGPR_32RegClass ||
626 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
627 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
628 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
633 if (RegClass == &AMDGPU::SReg_64RegClass ||
634 RegClass == &AMDGPU::SGPR_64RegClass ||
635 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
636 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
641 if (RegClass == &AMDGPU::VGPR_32RegClass) {
642 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
646 if (RegClass == &AMDGPU::VReg_64RegClass) {
647 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
652 unsigned EltSize = 4;
653 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
654 if (RI.isSGPRClass(RegClass)) {
655 if (RI.getRegSizeInBits(*RegClass) > 32) {
656 Opcode = AMDGPU::S_MOV_B64;
659 Opcode = AMDGPU::S_MOV_B32;
664 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
665 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
666 int64_t IdxValue = Idx == 0 ? Value : 0;
668 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
669 get(Opcode), RI.getSubReg(DestReg, Idx));
670 Builder.addImm(IdxValue);
674 const TargetRegisterClass *
675 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
676 return &AMDGPU::VGPR_32RegClass;
679 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
680 MachineBasicBlock::iterator I,
681 const DebugLoc &DL, unsigned DstReg,
682 ArrayRef<MachineOperand> Cond,
684 unsigned FalseReg) const {
685 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
686 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
689 if (Cond.size() == 1) {
690 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
691 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
693 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
697 } else if (Cond.size() == 2) {
698 assert(Cond[0].isImm() && "Cond[0] is not an immediate");
699 switch (Cond[0].getImm()) {
700 case SIInstrInfo::SCC_TRUE: {
701 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
702 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
705 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
711 case SIInstrInfo::SCC_FALSE: {
712 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
713 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
716 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
722 case SIInstrInfo::VCCNZ: {
723 MachineOperand RegOp = Cond[1];
724 RegOp.setImplicit(false);
725 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
726 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
728 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
734 case SIInstrInfo::VCCZ: {
735 MachineOperand RegOp = Cond[1];
736 RegOp.setImplicit(false);
737 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
738 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
740 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
746 case SIInstrInfo::EXECNZ: {
747 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
748 unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
749 BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
751 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
754 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
760 case SIInstrInfo::EXECZ: {
761 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
762 unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
763 BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
765 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
768 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
772 llvm_unreachable("Unhandled branch predicate EXECZ");
776 llvm_unreachable("invalid branch predicate");
779 llvm_unreachable("Can only handle Cond size 1 or 2");
783 unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
784 MachineBasicBlock::iterator I,
786 unsigned SrcReg, int Value) const {
787 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
788 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
789 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
796 unsigned SIInstrInfo::insertNE(MachineBasicBlock *MBB,
797 MachineBasicBlock::iterator I,
799 unsigned SrcReg, int Value) const {
800 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
801 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
802 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
809 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
811 if (RI.getRegSizeInBits(*DstRC) == 32) {
812 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
813 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
814 return AMDGPU::S_MOV_B64;
815 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
816 return AMDGPU::V_MOV_B64_PSEUDO;
821 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
824 return AMDGPU::SI_SPILL_S32_SAVE;
826 return AMDGPU::SI_SPILL_S64_SAVE;
828 return AMDGPU::SI_SPILL_S128_SAVE;
830 return AMDGPU::SI_SPILL_S256_SAVE;
832 return AMDGPU::SI_SPILL_S512_SAVE;
834 llvm_unreachable("unknown register size");
838 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
841 return AMDGPU::SI_SPILL_V32_SAVE;
843 return AMDGPU::SI_SPILL_V64_SAVE;
845 return AMDGPU::SI_SPILL_V96_SAVE;
847 return AMDGPU::SI_SPILL_V128_SAVE;
849 return AMDGPU::SI_SPILL_V256_SAVE;
851 return AMDGPU::SI_SPILL_V512_SAVE;
853 llvm_unreachable("unknown register size");
857 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
858 MachineBasicBlock::iterator MI,
859 unsigned SrcReg, bool isKill,
861 const TargetRegisterClass *RC,
862 const TargetRegisterInfo *TRI) const {
863 MachineFunction *MF = MBB.getParent();
864 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
865 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
866 DebugLoc DL = MBB.findDebugLoc(MI);
868 unsigned Size = FrameInfo.getObjectSize(FrameIndex);
869 unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
870 MachinePointerInfo PtrInfo
871 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
872 MachineMemOperand *MMO
873 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
875 unsigned SpillSize = TRI->getSpillSize(*RC);
877 if (RI.isSGPRClass(RC)) {
878 MFI->setHasSpilledSGPRs();
880 // We are only allowed to create one new instruction when spilling
881 // registers, so we need to use pseudo instruction for spilling SGPRs.
882 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
884 // The SGPR spill/restore instructions only work on number sgprs, so we need
885 // to make sure we are using the correct register class.
886 if (TargetRegisterInfo::isVirtualRegister(SrcReg) && SpillSize == 4) {
887 MachineRegisterInfo &MRI = MF->getRegInfo();
888 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
891 MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
892 .addReg(SrcReg, getKillRegState(isKill)) // data
893 .addFrameIndex(FrameIndex) // addr
895 .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
896 .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
897 // Add the scratch resource registers as implicit uses because we may end up
898 // needing them, and need to ensure that the reserved registers are
899 // correctly handled.
901 FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
902 if (ST.hasScalarStores()) {
903 // m0 is used for offset to scalar stores if used to spill.
904 Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
910 if (!ST.isVGPRSpillingEnabled(MF->getFunction())) {
911 LLVMContext &Ctx = MF->getFunction().getContext();
912 Ctx.emitError("SIInstrInfo::storeRegToStackSlot - Do not know how to"
914 BuildMI(MBB, MI, DL, get(AMDGPU::KILL))
920 assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
922 unsigned Opcode = getVGPRSpillSaveOpcode(SpillSize);
923 MFI->setHasSpilledVGPRs();
924 BuildMI(MBB, MI, DL, get(Opcode))
925 .addReg(SrcReg, getKillRegState(isKill)) // data
926 .addFrameIndex(FrameIndex) // addr
927 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
928 .addReg(MFI->getFrameOffsetReg()) // scratch_offset
933 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
936 return AMDGPU::SI_SPILL_S32_RESTORE;
938 return AMDGPU::SI_SPILL_S64_RESTORE;
940 return AMDGPU::SI_SPILL_S128_RESTORE;
942 return AMDGPU::SI_SPILL_S256_RESTORE;
944 return AMDGPU::SI_SPILL_S512_RESTORE;
946 llvm_unreachable("unknown register size");
950 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
953 return AMDGPU::SI_SPILL_V32_RESTORE;
955 return AMDGPU::SI_SPILL_V64_RESTORE;
957 return AMDGPU::SI_SPILL_V96_RESTORE;
959 return AMDGPU::SI_SPILL_V128_RESTORE;
961 return AMDGPU::SI_SPILL_V256_RESTORE;
963 return AMDGPU::SI_SPILL_V512_RESTORE;
965 llvm_unreachable("unknown register size");
969 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
970 MachineBasicBlock::iterator MI,
971 unsigned DestReg, int FrameIndex,
972 const TargetRegisterClass *RC,
973 const TargetRegisterInfo *TRI) const {
974 MachineFunction *MF = MBB.getParent();
975 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
976 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
977 DebugLoc DL = MBB.findDebugLoc(MI);
978 unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
979 unsigned Size = FrameInfo.getObjectSize(FrameIndex);
980 unsigned SpillSize = TRI->getSpillSize(*RC);
982 MachinePointerInfo PtrInfo
983 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
985 MachineMemOperand *MMO = MF->getMachineMemOperand(
986 PtrInfo, MachineMemOperand::MOLoad, Size, Align);
988 if (RI.isSGPRClass(RC)) {
989 // FIXME: Maybe this should not include a memoperand because it will be
990 // lowered to non-memory instructions.
991 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
992 if (TargetRegisterInfo::isVirtualRegister(DestReg) && SpillSize == 4) {
993 MachineRegisterInfo &MRI = MF->getRegInfo();
994 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
997 FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
998 MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
999 .addFrameIndex(FrameIndex) // addr
1001 .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1002 .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
1004 if (ST.hasScalarStores()) {
1005 // m0 is used for offset to scalar stores if used to spill.
1006 Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1012 if (!ST.isVGPRSpillingEnabled(MF->getFunction())) {
1013 LLVMContext &Ctx = MF->getFunction().getContext();
1014 Ctx.emitError("SIInstrInfo::loadRegFromStackSlot - Do not know how to"
1015 " restore register");
1016 BuildMI(MBB, MI, DL, get(AMDGPU::IMPLICIT_DEF), DestReg);
1021 assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
1023 unsigned Opcode = getVGPRSpillRestoreOpcode(SpillSize);
1024 BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1025 .addFrameIndex(FrameIndex) // vaddr
1026 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1027 .addReg(MFI->getFrameOffsetReg()) // scratch_offset
1028 .addImm(0) // offset
1029 .addMemOperand(MMO);
1032 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1033 unsigned SIInstrInfo::calculateLDSSpillAddress(
1034 MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1035 unsigned FrameOffset, unsigned Size) const {
1036 MachineFunction *MF = MBB.getParent();
1037 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1038 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1039 DebugLoc DL = MBB.findDebugLoc(MI);
1040 unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1041 unsigned WavefrontSize = ST.getWavefrontSize();
1043 unsigned TIDReg = MFI->getTIDReg();
1044 if (!MFI->hasCalculatedTID()) {
1045 MachineBasicBlock &Entry = MBB.getParent()->front();
1046 MachineBasicBlock::iterator Insert = Entry.front();
1047 DebugLoc DL = Insert->getDebugLoc();
1049 TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1051 if (TIDReg == AMDGPU::NoRegister)
1054 if (!AMDGPU::isShader(MF->getFunction().getCallingConv()) &&
1055 WorkGroupSize > WavefrontSize) {
1057 = MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
1059 = MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
1061 = MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
1062 unsigned InputPtrReg =
1063 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1064 for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1065 if (!Entry.isLiveIn(Reg))
1066 Entry.addLiveIn(Reg);
1069 RS->enterBasicBlock(Entry);
1070 // FIXME: Can we scavenge an SReg_64 and access the subregs?
1071 unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1072 unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1073 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1074 .addReg(InputPtrReg)
1075 .addImm(SI::KernelInputOffsets::NGROUPS_Z);
1076 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1077 .addReg(InputPtrReg)
1078 .addImm(SI::KernelInputOffsets::NGROUPS_Y);
1080 // NGROUPS.X * NGROUPS.Y
1081 BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1084 // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1085 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1088 // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1089 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1093 // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1094 getAddNoCarry(Entry, Insert, DL, TIDReg)
1099 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1104 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1110 BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1114 MFI->setTIDReg(TIDReg);
1117 // Add FrameIndex to LDS offset
1118 unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1119 getAddNoCarry(MBB, MI, DL, TmpReg)
1126 void SIInstrInfo::insertWaitStates(MachineBasicBlock &MBB,
1127 MachineBasicBlock::iterator MI,
1129 DebugLoc DL = MBB.findDebugLoc(MI);
1137 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1142 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1143 MachineBasicBlock::iterator MI) const {
1144 insertWaitStates(MBB, MI, 1);
1147 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
1148 auto MF = MBB.getParent();
1149 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1151 assert(Info->isEntryFunction());
1153 if (MBB.succ_empty()) {
1154 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1155 if (HasNoTerminator)
1156 BuildMI(MBB, MBB.end(), DebugLoc(),
1157 get(Info->returnsVoid() ? AMDGPU::S_ENDPGM : AMDGPU::SI_RETURN_TO_EPILOG));
1161 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) const {
1162 switch (MI.getOpcode()) {
1163 default: return 1; // FIXME: Do wait states equal cycles?
1166 return MI.getOperand(0).getImm() + 1;
1170 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1171 MachineBasicBlock &MBB = *MI.getParent();
1172 DebugLoc DL = MBB.findDebugLoc(MI);
1173 switch (MI.getOpcode()) {
1174 default: return TargetInstrInfo::expandPostRAPseudo(MI);
1175 case AMDGPU::S_MOV_B64_term:
1176 // This is only a terminator to get the correct spill code placement during
1177 // register allocation.
1178 MI.setDesc(get(AMDGPU::S_MOV_B64));
1181 case AMDGPU::S_XOR_B64_term:
1182 // This is only a terminator to get the correct spill code placement during
1183 // register allocation.
1184 MI.setDesc(get(AMDGPU::S_XOR_B64));
1187 case AMDGPU::S_ANDN2_B64_term:
1188 // This is only a terminator to get the correct spill code placement during
1189 // register allocation.
1190 MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1193 case AMDGPU::V_MOV_B64_PSEUDO: {
1194 unsigned Dst = MI.getOperand(0).getReg();
1195 unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1196 unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1198 const MachineOperand &SrcOp = MI.getOperand(1);
1199 // FIXME: Will this work for 64-bit floating point immediates?
1200 assert(!SrcOp.isFPImm());
1201 if (SrcOp.isImm()) {
1202 APInt Imm(64, SrcOp.getImm());
1203 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1204 .addImm(Imm.getLoBits(32).getZExtValue())
1205 .addReg(Dst, RegState::Implicit | RegState::Define);
1206 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1207 .addImm(Imm.getHiBits(32).getZExtValue())
1208 .addReg(Dst, RegState::Implicit | RegState::Define);
1210 assert(SrcOp.isReg());
1211 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1212 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1213 .addReg(Dst, RegState::Implicit | RegState::Define);
1214 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1215 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1216 .addReg(Dst, RegState::Implicit | RegState::Define);
1218 MI.eraseFromParent();
1221 case AMDGPU::V_SET_INACTIVE_B32: {
1222 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1223 .addReg(AMDGPU::EXEC);
1224 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1225 .add(MI.getOperand(2));
1226 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1227 .addReg(AMDGPU::EXEC);
1228 MI.eraseFromParent();
1231 case AMDGPU::V_SET_INACTIVE_B64: {
1232 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1233 .addReg(AMDGPU::EXEC);
1234 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1235 MI.getOperand(0).getReg())
1236 .add(MI.getOperand(2));
1237 expandPostRAPseudo(*Copy);
1238 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1239 .addReg(AMDGPU::EXEC);
1240 MI.eraseFromParent();
1243 case AMDGPU::V_MOVRELD_B32_V1:
1244 case AMDGPU::V_MOVRELD_B32_V2:
1245 case AMDGPU::V_MOVRELD_B32_V4:
1246 case AMDGPU::V_MOVRELD_B32_V8:
1247 case AMDGPU::V_MOVRELD_B32_V16: {
1248 const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1249 unsigned VecReg = MI.getOperand(0).getReg();
1250 bool IsUndef = MI.getOperand(1).isUndef();
1251 unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1252 assert(VecReg == MI.getOperand(1).getReg());
1254 MachineInstr *MovRel =
1255 BuildMI(MBB, MI, DL, MovRelDesc)
1256 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1257 .add(MI.getOperand(2))
1258 .addReg(VecReg, RegState::ImplicitDefine)
1260 RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1262 const int ImpDefIdx =
1263 MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1264 const int ImpUseIdx = ImpDefIdx + 1;
1265 MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1267 MI.eraseFromParent();
1270 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1271 MachineFunction &MF = *MBB.getParent();
1272 unsigned Reg = MI.getOperand(0).getReg();
1273 unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1274 unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1276 // Create a bundle so these instructions won't be re-ordered by the
1277 // post-RA scheduler.
1278 MIBundleBuilder Bundler(MBB, MI);
1279 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1281 // Add 32-bit offset from this instruction to the start of the
1283 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1285 .add(MI.getOperand(1)));
1287 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1289 if (MI.getOperand(2).getTargetFlags() == SIInstrInfo::MO_NONE)
1292 MIB.add(MI.getOperand(2));
1294 Bundler.append(MIB);
1295 finalizeBundle(MBB, Bundler.begin());
1297 MI.eraseFromParent();
1300 case AMDGPU::EXIT_WWM: {
1301 // This only gets its own opcode so that SIFixWWMLiveness can tell when WWM
1303 MI.setDesc(get(AMDGPU::S_MOV_B64));
1306 case TargetOpcode::BUNDLE: {
1310 // If it is a load it must be a memory clause
1311 for (MachineBasicBlock::instr_iterator I = MI.getIterator();
1312 I->isBundledWithSucc(); ++I) {
1313 I->unbundleFromSucc();
1314 for (MachineOperand &MO : I->operands())
1316 MO.setIsInternalRead(false);
1319 MI.eraseFromParent();
1326 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
1327 MachineOperand &Src0,
1328 unsigned Src0OpName,
1329 MachineOperand &Src1,
1330 unsigned Src1OpName) const {
1331 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1335 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1337 "All commutable instructions have both src0 and src1 modifiers");
1339 int Src0ModsVal = Src0Mods->getImm();
1340 int Src1ModsVal = Src1Mods->getImm();
1342 Src1Mods->setImm(Src0ModsVal);
1343 Src0Mods->setImm(Src1ModsVal);
1347 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
1348 MachineOperand &RegOp,
1349 MachineOperand &NonRegOp) {
1350 unsigned Reg = RegOp.getReg();
1351 unsigned SubReg = RegOp.getSubReg();
1352 bool IsKill = RegOp.isKill();
1353 bool IsDead = RegOp.isDead();
1354 bool IsUndef = RegOp.isUndef();
1355 bool IsDebug = RegOp.isDebug();
1357 if (NonRegOp.isImm())
1358 RegOp.ChangeToImmediate(NonRegOp.getImm());
1359 else if (NonRegOp.isFI())
1360 RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1364 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1365 NonRegOp.setSubReg(SubReg);
1370 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1372 unsigned Src1Idx) const {
1373 assert(!NewMI && "this should never be used");
1375 unsigned Opc = MI.getOpcode();
1376 int CommutedOpcode = commuteOpcode(Opc);
1377 if (CommutedOpcode == -1)
1380 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1381 static_cast<int>(Src0Idx) &&
1382 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1383 static_cast<int>(Src1Idx) &&
1384 "inconsistency with findCommutedOpIndices");
1386 MachineOperand &Src0 = MI.getOperand(Src0Idx);
1387 MachineOperand &Src1 = MI.getOperand(Src1Idx);
1389 MachineInstr *CommutedMI = nullptr;
1390 if (Src0.isReg() && Src1.isReg()) {
1391 if (isOperandLegal(MI, Src1Idx, &Src0)) {
1392 // Be sure to copy the source modifiers to the right place.
1394 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1397 } else if (Src0.isReg() && !Src1.isReg()) {
1398 // src0 should always be able to support any operand type, so no need to
1399 // check operand legality.
1400 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1401 } else if (!Src0.isReg() && Src1.isReg()) {
1402 if (isOperandLegal(MI, Src1Idx, &Src0))
1403 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1405 // FIXME: Found two non registers to commute. This does happen.
1410 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1411 Src1, AMDGPU::OpName::src1_modifiers);
1413 CommutedMI->setDesc(get(CommutedOpcode));
1419 // This needs to be implemented because the source modifiers may be inserted
1420 // between the true commutable operands, and the base
1421 // TargetInstrInfo::commuteInstruction uses it.
1422 bool SIInstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx0,
1423 unsigned &SrcOpIdx1) const {
1424 if (!MI.isCommutable())
1427 unsigned Opc = MI.getOpcode();
1428 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1432 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1436 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1439 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1440 int64_t BrOffset) const {
1441 // BranchRelaxation should never have to check s_setpc_b64 because its dest
1442 // block is unanalyzable.
1443 assert(BranchOp != AMDGPU::S_SETPC_B64);
1445 // Convert to dwords.
1448 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1449 // from the next instruction.
1452 return isIntN(BranchOffsetBits, BrOffset);
1455 MachineBasicBlock *SIInstrInfo::getBranchDestBlock(
1456 const MachineInstr &MI) const {
1457 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1458 // This would be a difficult analysis to perform, but can always be legal so
1459 // there's no need to analyze it.
1463 return MI.getOperand(0).getMBB();
1466 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1467 MachineBasicBlock &DestBB,
1470 RegScavenger *RS) const {
1471 assert(RS && "RegScavenger required for long branching");
1472 assert(MBB.empty() &&
1473 "new block should be inserted for expanding unconditional branch");
1474 assert(MBB.pred_size() == 1);
1476 MachineFunction *MF = MBB.getParent();
1477 MachineRegisterInfo &MRI = MF->getRegInfo();
1479 // FIXME: Virtual register workaround for RegScavenger not working with empty
1481 unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1485 // We need to compute the offset relative to the instruction immediately after
1486 // s_getpc_b64. Insert pc arithmetic code before last terminator.
1487 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1489 // TODO: Handle > 32-bit block address.
1490 if (BrOffset >= 0) {
1491 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1492 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1493 .addReg(PCReg, 0, AMDGPU::sub0)
1494 .addMBB(&DestBB, AMDGPU::TF_LONG_BRANCH_FORWARD);
1495 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1496 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1497 .addReg(PCReg, 0, AMDGPU::sub1)
1500 // Backwards branch.
1501 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1502 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1503 .addReg(PCReg, 0, AMDGPU::sub0)
1504 .addMBB(&DestBB, AMDGPU::TF_LONG_BRANCH_BACKWARD);
1505 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1506 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1507 .addReg(PCReg, 0, AMDGPU::sub1)
1511 // Insert the indirect branch after the other terminator.
1512 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1515 // FIXME: If spilling is necessary, this will fail because this scavenger has
1516 // no emergency stack slots. It is non-trivial to spill in this situation,
1517 // because the restore code needs to be specially placed after the
1518 // jump. BranchRelaxation then needs to be made aware of the newly inserted
1521 // If a spill is needed for the pc register pair, we need to insert a spill
1522 // restore block right before the destination block, and insert a short branch
1523 // into the old destination block's fallthrough predecessor.
1526 // s_cbranch_scc0 skip_long_branch:
1530 // s_getpc_b64 s[8:9]
1531 // s_add_u32 s8, s8, restore_bb
1532 // s_addc_u32 s9, s9, 0
1533 // s_setpc_b64 s[8:9]
1535 // skip_long_branch:
1540 // dest_bb_fallthrough_predecessor:
1546 // fallthrough dest_bb
1551 RS->enterBasicBlockEnd(MBB);
1552 unsigned Scav = RS->scavengeRegister(&AMDGPU::SReg_64RegClass,
1553 MachineBasicBlock::iterator(GetPC), 0);
1554 MRI.replaceRegWith(PCReg, Scav);
1555 MRI.clearVirtRegs();
1556 RS->setRegUsed(Scav);
1558 return 4 + 8 + 4 + 4;
1561 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1563 case SIInstrInfo::SCC_TRUE:
1564 return AMDGPU::S_CBRANCH_SCC1;
1565 case SIInstrInfo::SCC_FALSE:
1566 return AMDGPU::S_CBRANCH_SCC0;
1567 case SIInstrInfo::VCCNZ:
1568 return AMDGPU::S_CBRANCH_VCCNZ;
1569 case SIInstrInfo::VCCZ:
1570 return AMDGPU::S_CBRANCH_VCCZ;
1571 case SIInstrInfo::EXECNZ:
1572 return AMDGPU::S_CBRANCH_EXECNZ;
1573 case SIInstrInfo::EXECZ:
1574 return AMDGPU::S_CBRANCH_EXECZ;
1576 llvm_unreachable("invalid branch predicate");
1580 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1582 case AMDGPU::S_CBRANCH_SCC0:
1584 case AMDGPU::S_CBRANCH_SCC1:
1586 case AMDGPU::S_CBRANCH_VCCNZ:
1588 case AMDGPU::S_CBRANCH_VCCZ:
1590 case AMDGPU::S_CBRANCH_EXECNZ:
1592 case AMDGPU::S_CBRANCH_EXECZ:
1599 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
1600 MachineBasicBlock::iterator I,
1601 MachineBasicBlock *&TBB,
1602 MachineBasicBlock *&FBB,
1603 SmallVectorImpl<MachineOperand> &Cond,
1604 bool AllowModify) const {
1605 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1606 // Unconditional Branch
1607 TBB = I->getOperand(0).getMBB();
1611 MachineBasicBlock *CondBB = nullptr;
1613 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1614 CondBB = I->getOperand(1).getMBB();
1615 Cond.push_back(I->getOperand(0));
1617 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1618 if (Pred == INVALID_BR)
1621 CondBB = I->getOperand(0).getMBB();
1622 Cond.push_back(MachineOperand::CreateImm(Pred));
1623 Cond.push_back(I->getOperand(1)); // Save the branch register.
1627 if (I == MBB.end()) {
1628 // Conditional branch followed by fall-through.
1633 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1635 FBB = I->getOperand(0).getMBB();
1642 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
1643 MachineBasicBlock *&FBB,
1644 SmallVectorImpl<MachineOperand> &Cond,
1645 bool AllowModify) const {
1646 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1650 if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1651 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1655 // TODO: Should be able to treat as fallthrough?
1659 if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1662 MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1664 // Specifically handle the case where the conditional branch is to the same
1665 // destination as the mask branch. e.g.
1667 // si_mask_branch BB8
1668 // s_cbranch_execz BB8
1671 // This is required to understand divergent loops which may need the branches
1673 if (TBB != MaskBrDest || Cond.empty())
1676 auto Pred = Cond[0].getImm();
1677 return (Pred != EXECZ && Pred != EXECNZ);
1680 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
1681 int *BytesRemoved) const {
1682 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1685 unsigned RemovedSize = 0;
1686 while (I != MBB.end()) {
1687 MachineBasicBlock::iterator Next = std::next(I);
1688 if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1693 RemovedSize += getInstSizeInBytes(*I);
1694 I->eraseFromParent();
1700 *BytesRemoved = RemovedSize;
1705 // Copy the flags onto the implicit condition register operand.
1706 static void preserveCondRegFlags(MachineOperand &CondReg,
1707 const MachineOperand &OrigCond) {
1708 CondReg.setIsUndef(OrigCond.isUndef());
1709 CondReg.setIsKill(OrigCond.isKill());
1712 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
1713 MachineBasicBlock *TBB,
1714 MachineBasicBlock *FBB,
1715 ArrayRef<MachineOperand> Cond,
1717 int *BytesAdded) const {
1718 if (!FBB && Cond.empty()) {
1719 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1726 if(Cond.size() == 1 && Cond[0].isReg()) {
1727 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1733 assert(TBB && Cond[0].isImm());
1736 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
1740 MachineInstr *CondBr =
1741 BuildMI(&MBB, DL, get(Opcode))
1744 // Copy the flags onto the implicit condition register operand.
1745 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
1754 MachineInstr *CondBr =
1755 BuildMI(&MBB, DL, get(Opcode))
1757 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1760 MachineOperand &CondReg = CondBr->getOperand(1);
1761 CondReg.setIsUndef(Cond[1].isUndef());
1762 CondReg.setIsKill(Cond[1].isKill());
1770 bool SIInstrInfo::reverseBranchCondition(
1771 SmallVectorImpl<MachineOperand> &Cond) const {
1772 if (Cond.size() != 2) {
1776 if (Cond[0].isImm()) {
1777 Cond[0].setImm(-Cond[0].getImm());
1784 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1785 ArrayRef<MachineOperand> Cond,
1786 unsigned TrueReg, unsigned FalseReg,
1788 int &TrueCycles, int &FalseCycles) const {
1789 switch (Cond[0].getImm()) {
1792 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1793 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1794 assert(MRI.getRegClass(FalseReg) == RC);
1796 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1797 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1799 // Limit to equal cost for branch vs. N v_cndmask_b32s.
1800 return !RI.isSGPRClass(RC) && NumInsts <= 6;
1804 // FIXME: We could insert for VGPRs if we could replace the original compare
1805 // with a vector one.
1806 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1807 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1808 assert(MRI.getRegClass(FalseReg) == RC);
1810 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1812 // Multiples of 8 can do s_cselect_b64
1813 if (NumInsts % 2 == 0)
1816 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1817 return RI.isSGPRClass(RC);
1824 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
1825 MachineBasicBlock::iterator I, const DebugLoc &DL,
1826 unsigned DstReg, ArrayRef<MachineOperand> Cond,
1827 unsigned TrueReg, unsigned FalseReg) const {
1828 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
1829 if (Pred == VCCZ || Pred == SCC_FALSE) {
1830 Pred = static_cast<BranchPredicate>(-Pred);
1831 std::swap(TrueReg, FalseReg);
1834 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1835 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
1836 unsigned DstSize = RI.getRegSizeInBits(*DstRC);
1838 if (DstSize == 32) {
1839 unsigned SelOp = Pred == SCC_TRUE ?
1840 AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
1842 // Instruction's operands are backwards from what is expected.
1843 MachineInstr *Select =
1844 BuildMI(MBB, I, DL, get(SelOp), DstReg)
1848 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1852 if (DstSize == 64 && Pred == SCC_TRUE) {
1853 MachineInstr *Select =
1854 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
1858 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1862 static const int16_t Sub0_15[] = {
1863 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1864 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1865 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
1866 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
1869 static const int16_t Sub0_15_64[] = {
1870 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1871 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
1872 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
1873 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
1876 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
1877 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
1878 const int16_t *SubIndices = Sub0_15;
1879 int NElts = DstSize / 32;
1881 // 64-bit select is only avaialble for SALU.
1882 if (Pred == SCC_TRUE) {
1883 SelOp = AMDGPU::S_CSELECT_B64;
1884 EltRC = &AMDGPU::SGPR_64RegClass;
1885 SubIndices = Sub0_15_64;
1887 assert(NElts % 2 == 0);
1891 MachineInstrBuilder MIB = BuildMI(
1892 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
1894 I = MIB->getIterator();
1896 SmallVector<unsigned, 8> Regs;
1897 for (int Idx = 0; Idx != NElts; ++Idx) {
1898 unsigned DstElt = MRI.createVirtualRegister(EltRC);
1899 Regs.push_back(DstElt);
1901 unsigned SubIdx = SubIndices[Idx];
1903 MachineInstr *Select =
1904 BuildMI(MBB, I, DL, get(SelOp), DstElt)
1905 .addReg(FalseReg, 0, SubIdx)
1906 .addReg(TrueReg, 0, SubIdx);
1907 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1914 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const {
1915 switch (MI.getOpcode()) {
1916 case AMDGPU::V_MOV_B32_e32:
1917 case AMDGPU::V_MOV_B32_e64:
1918 case AMDGPU::V_MOV_B64_PSEUDO: {
1919 // If there are additional implicit register operands, this may be used for
1920 // register indexing so the source register operand isn't simply copied.
1921 unsigned NumOps = MI.getDesc().getNumOperands() +
1922 MI.getDesc().getNumImplicitUses();
1924 return MI.getNumOperands() == NumOps;
1926 case AMDGPU::S_MOV_B32:
1927 case AMDGPU::S_MOV_B64:
1935 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind(
1936 PseudoSourceValue::PSVKind Kind) const {
1938 case PseudoSourceValue::Stack:
1939 case PseudoSourceValue::FixedStack:
1940 return ST.getAMDGPUAS().PRIVATE_ADDRESS;
1941 case PseudoSourceValue::ConstantPool:
1942 case PseudoSourceValue::GOT:
1943 case PseudoSourceValue::JumpTable:
1944 case PseudoSourceValue::GlobalValueCallEntry:
1945 case PseudoSourceValue::ExternalSymbolCallEntry:
1946 case PseudoSourceValue::TargetCustom:
1947 return ST.getAMDGPUAS().CONSTANT_ADDRESS;
1949 return ST.getAMDGPUAS().FLAT_ADDRESS;
1952 static void removeModOperands(MachineInstr &MI) {
1953 unsigned Opc = MI.getOpcode();
1954 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1955 AMDGPU::OpName::src0_modifiers);
1956 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1957 AMDGPU::OpName::src1_modifiers);
1958 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1959 AMDGPU::OpName::src2_modifiers);
1961 MI.RemoveOperand(Src2ModIdx);
1962 MI.RemoveOperand(Src1ModIdx);
1963 MI.RemoveOperand(Src0ModIdx);
1966 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1967 unsigned Reg, MachineRegisterInfo *MRI) const {
1968 if (!MRI->hasOneNonDBGUse(Reg))
1971 switch (DefMI.getOpcode()) {
1974 case AMDGPU::S_MOV_B64:
1975 // TODO: We could fold 64-bit immediates, but this get compilicated
1976 // when there are sub-registers.
1979 case AMDGPU::V_MOV_B32_e32:
1980 case AMDGPU::S_MOV_B32:
1984 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
1986 // FIXME: We could handle FrameIndex values here.
1987 if (!ImmOp->isImm())
1990 unsigned Opc = UseMI.getOpcode();
1991 if (Opc == AMDGPU::COPY) {
1992 bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
1993 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
1994 UseMI.setDesc(get(NewOpc));
1995 UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
1996 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2000 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2001 Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64) {
2002 // Don't fold if we are using source or output modifiers. The new VOP2
2003 // instructions don't have them.
2004 if (hasAnyModifiersSet(UseMI))
2007 // If this is a free constant, there's no reason to do this.
2008 // TODO: We could fold this here instead of letting SIFoldOperands do it
2010 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2012 // Any src operand can be used for the legality check.
2013 if (isInlineConstant(UseMI, *Src0, *ImmOp))
2016 bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64;
2017 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2018 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2020 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2021 // We should only expect these to be on src0 due to canonicalizations.
2022 if (Src0->isReg() && Src0->getReg() == Reg) {
2023 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2026 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2029 // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2031 const int64_t Imm = ImmOp->getImm();
2033 // FIXME: This would be a lot easier if we could return a new instruction
2034 // instead of having to modify in place.
2036 // Remove these first since they are at the end.
2037 UseMI.RemoveOperand(
2038 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2039 UseMI.RemoveOperand(
2040 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2042 unsigned Src1Reg = Src1->getReg();
2043 unsigned Src1SubReg = Src1->getSubReg();
2044 Src0->setReg(Src1Reg);
2045 Src0->setSubReg(Src1SubReg);
2046 Src0->setIsKill(Src1->isKill());
2048 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2049 Opc == AMDGPU::V_MAC_F16_e64)
2050 UseMI.untieRegOperand(
2051 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2053 Src1->ChangeToImmediate(Imm);
2055 removeModOperands(UseMI);
2056 UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16));
2058 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2060 DefMI.eraseFromParent();
2065 // Added part is the constant: Use v_madak_{f16, f32}.
2066 if (Src2->isReg() && Src2->getReg() == Reg) {
2067 // Not allowed to use constant bus for another operand.
2068 // We can however allow an inline immediate as src0.
2069 if (!Src0->isImm() &&
2070 (Src0->isReg() && RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))
2073 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2076 const int64_t Imm = ImmOp->getImm();
2078 // FIXME: This would be a lot easier if we could return a new instruction
2079 // instead of having to modify in place.
2081 // Remove these first since they are at the end.
2082 UseMI.RemoveOperand(
2083 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2084 UseMI.RemoveOperand(
2085 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2087 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2088 Opc == AMDGPU::V_MAC_F16_e64)
2089 UseMI.untieRegOperand(
2090 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2092 // ChangingToImmediate adds Src2 back to the instruction.
2093 Src2->ChangeToImmediate(Imm);
2095 // These come before src2.
2096 removeModOperands(UseMI);
2097 UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16));
2099 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2101 DefMI.eraseFromParent();
2110 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2111 int WidthB, int OffsetB) {
2112 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2113 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2114 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2115 return LowOffset + LowWidth <= HighOffset;
2118 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr &MIa,
2119 MachineInstr &MIb) const {
2120 unsigned BaseReg0, BaseReg1;
2121 int64_t Offset0, Offset1;
2123 if (getMemOpBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) &&
2124 getMemOpBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) {
2126 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2127 // FIXME: Handle ds_read2 / ds_write2.
2130 unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2131 unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2132 if (BaseReg0 == BaseReg1 &&
2133 offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2141 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr &MIa,
2143 AliasAnalysis *AA) const {
2144 assert((MIa.mayLoad() || MIa.mayStore()) &&
2145 "MIa must load from or modify a memory location");
2146 assert((MIb.mayLoad() || MIb.mayStore()) &&
2147 "MIb must load from or modify a memory location");
2149 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
2152 // XXX - Can we relax this between address spaces?
2153 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2156 if (AA && MIa.hasOneMemOperand() && MIb.hasOneMemOperand()) {
2157 const MachineMemOperand *MMOa = *MIa.memoperands_begin();
2158 const MachineMemOperand *MMOb = *MIb.memoperands_begin();
2159 if (MMOa->getValue() && MMOb->getValue()) {
2160 MemoryLocation LocA(MMOa->getValue(), MMOa->getSize(), MMOa->getAAInfo());
2161 MemoryLocation LocB(MMOb->getValue(), MMOb->getSize(), MMOb->getAAInfo());
2162 if (!AA->alias(LocA, LocB))
2167 // TODO: Should we check the address space from the MachineMemOperand? That
2168 // would allow us to distinguish objects we know don't alias based on the
2169 // underlying address space, even if it was lowered to a different one,
2170 // e.g. private accesses lowered to use MUBUF instructions on a scratch
2174 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2176 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2179 if (isMUBUF(MIa) || isMTBUF(MIa)) {
2180 if (isMUBUF(MIb) || isMTBUF(MIb))
2181 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2183 return !isFLAT(MIb) && !isSMRD(MIb);
2188 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2190 return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2195 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2203 static int64_t getFoldableImm(const MachineOperand* MO) {
2206 const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2207 const MachineRegisterInfo &MRI = MF->getRegInfo();
2208 auto Def = MRI.getUniqueVRegDef(MO->getReg());
2209 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2210 Def->getOperand(1).isImm())
2211 return Def->getOperand(1).getImm();
2212 return AMDGPU::NoRegister;
2215 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
2217 LiveVariables *LV) const {
2218 unsigned Opc = MI.getOpcode();
2220 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64;
2225 case AMDGPU::V_MAC_F16_e64:
2228 case AMDGPU::V_MAC_F32_e64:
2229 case AMDGPU::V_FMAC_F32_e64:
2231 case AMDGPU::V_MAC_F16_e32:
2234 case AMDGPU::V_MAC_F32_e32:
2235 case AMDGPU::V_FMAC_F32_e32: {
2236 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2237 AMDGPU::OpName::src0);
2238 const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2239 if (!Src0->isReg() && !Src0->isImm())
2242 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2249 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2250 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2251 const MachineOperand *Src0Mods =
2252 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2253 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2254 const MachineOperand *Src1Mods =
2255 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2256 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2257 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2258 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2260 if (!IsFMA && !Src0Mods && !Src1Mods && !Clamp && !Omod &&
2261 // If we have an SGPR input, we will violate the constant bus restriction.
2262 (!Src0->isReg() || !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2263 if (auto Imm = getFoldableImm(Src2)) {
2264 return BuildMI(*MBB, MI, MI.getDebugLoc(),
2265 get(IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32))
2271 if (auto Imm = getFoldableImm(Src1)) {
2272 return BuildMI(*MBB, MI, MI.getDebugLoc(),
2273 get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2279 if (auto Imm = getFoldableImm(Src0)) {
2280 if (isOperandLegal(MI, AMDGPU::getNamedOperandIdx(AMDGPU::V_MADMK_F32,
2281 AMDGPU::OpName::src0), Src1))
2282 return BuildMI(*MBB, MI, MI.getDebugLoc(),
2283 get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2291 assert((!IsFMA || !IsF16) && "fmac only expected with f32");
2292 unsigned NewOpc = IsFMA ? AMDGPU::V_FMA_F32 :
2293 (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2294 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2296 .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2298 .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2300 .addImm(0) // Src mods
2302 .addImm(Clamp ? Clamp->getImm() : 0)
2303 .addImm(Omod ? Omod->getImm() : 0);
2306 // It's not generally safe to move VALU instructions across these since it will
2307 // start using the register as a base index rather than directly.
2308 // XXX - Why isn't hasSideEffects sufficient for these?
2309 static bool changesVGPRIndexingMode(const MachineInstr &MI) {
2310 switch (MI.getOpcode()) {
2311 case AMDGPU::S_SET_GPR_IDX_ON:
2312 case AMDGPU::S_SET_GPR_IDX_MODE:
2313 case AMDGPU::S_SET_GPR_IDX_OFF:
2320 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2321 const MachineBasicBlock *MBB,
2322 const MachineFunction &MF) const {
2323 // XXX - Do we want the SP check in the base implementation?
2325 // Target-independent instructions do not have an implicit-use of EXEC, even
2326 // when they operate on VGPRs. Treating EXEC modifications as scheduling
2327 // boundaries prevents incorrect movements of such instructions.
2328 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2329 MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2330 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2331 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2332 changesVGPRIndexingMode(MI);
2335 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
2336 unsigned Opcode = MI.getOpcode();
2338 if (MI.mayStore() && isSMRD(MI))
2339 return true; // scalar store or atomic
2341 // These instructions cause shader I/O that may cause hardware lockups
2342 // when executed with an empty EXEC mask.
2344 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2345 // EXEC = 0, but checking for that case here seems not worth it
2346 // given the typical code patterns.
2347 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2348 Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE)
2351 if (MI.isInlineAsm())
2352 return true; // conservative assumption
2354 // These are like SALU instructions in terms of effects, so it's questionable
2355 // whether we should return true for those.
2357 // However, executing them with EXEC = 0 causes them to operate on undefined
2358 // data, which we avoid by returning true here.
2359 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2365 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2366 switch (Imm.getBitWidth()) {
2368 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
2369 ST.hasInv2PiInlineImm());
2371 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
2372 ST.hasInv2PiInlineImm());
2374 return ST.has16BitInsts() &&
2375 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
2376 ST.hasInv2PiInlineImm());
2378 llvm_unreachable("invalid bitwidth");
2382 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
2383 uint8_t OperandType) const {
2385 OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2386 OperandType > AMDGPU::OPERAND_SRC_LAST)
2389 // MachineOperand provides no way to tell the true operand size, since it only
2390 // records a 64-bit value. We need to know the size to determine if a 32-bit
2391 // floating point immediate bit pattern is legal for an integer immediate. It
2392 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2394 int64_t Imm = MO.getImm();
2395 switch (OperandType) {
2396 case AMDGPU::OPERAND_REG_IMM_INT32:
2397 case AMDGPU::OPERAND_REG_IMM_FP32:
2398 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2399 case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
2400 int32_t Trunc = static_cast<int32_t>(Imm);
2401 return Trunc == Imm &&
2402 AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
2404 case AMDGPU::OPERAND_REG_IMM_INT64:
2405 case AMDGPU::OPERAND_REG_IMM_FP64:
2406 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2407 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2408 return AMDGPU::isInlinableLiteral64(MO.getImm(),
2409 ST.hasInv2PiInlineImm());
2410 case AMDGPU::OPERAND_REG_IMM_INT16:
2411 case AMDGPU::OPERAND_REG_IMM_FP16:
2412 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2413 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
2414 if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2415 // A few special case instructions have 16-bit operands on subtargets
2416 // where 16-bit instructions are not legal.
2417 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2418 // constants in these cases
2419 int16_t Trunc = static_cast<int16_t>(Imm);
2420 return ST.has16BitInsts() &&
2421 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
2426 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2427 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
2428 if (isUInt<16>(Imm)) {
2429 int16_t Trunc = static_cast<int16_t>(Imm);
2430 return ST.has16BitInsts() &&
2431 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
2433 if (!(Imm & 0xffff)) {
2434 return ST.has16BitInsts() &&
2435 AMDGPU::isInlinableLiteral16(Imm >> 16, ST.hasInv2PiInlineImm());
2437 uint32_t Trunc = static_cast<uint32_t>(Imm);
2438 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm());
2441 llvm_unreachable("invalid bitwidth");
2445 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO,
2446 const MCOperandInfo &OpInfo) const {
2447 switch (MO.getType()) {
2448 case MachineOperand::MO_Register:
2450 case MachineOperand::MO_Immediate:
2451 return !isInlineConstant(MO, OpInfo);
2452 case MachineOperand::MO_FrameIndex:
2453 case MachineOperand::MO_MachineBasicBlock:
2454 case MachineOperand::MO_ExternalSymbol:
2455 case MachineOperand::MO_GlobalAddress:
2456 case MachineOperand::MO_MCSymbol:
2459 llvm_unreachable("unexpected operand type");
2463 static bool compareMachineOp(const MachineOperand &Op0,
2464 const MachineOperand &Op1) {
2465 if (Op0.getType() != Op1.getType())
2468 switch (Op0.getType()) {
2469 case MachineOperand::MO_Register:
2470 return Op0.getReg() == Op1.getReg();
2471 case MachineOperand::MO_Immediate:
2472 return Op0.getImm() == Op1.getImm();
2474 llvm_unreachable("Didn't expect to be comparing these operand types");
2478 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
2479 const MachineOperand &MO) const {
2480 const MCOperandInfo &OpInfo = get(MI.getOpcode()).OpInfo[OpNo];
2482 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
2484 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2487 if (OpInfo.RegClass < 0)
2490 if (MO.isImm() && isInlineConstant(MO, OpInfo))
2491 return RI.opCanUseInlineConstant(OpInfo.OperandType);
2493 return RI.opCanUseLiteralConstant(OpInfo.OperandType);
2496 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2497 int Op32 = AMDGPU::getVOPe32(Opcode);
2501 return pseudoToMCOpcode(Op32) != -1;
2504 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2505 // The src0_modifier operand is present on all instructions
2506 // that have modifiers.
2508 return AMDGPU::getNamedOperandIdx(Opcode,
2509 AMDGPU::OpName::src0_modifiers) != -1;
2512 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
2513 unsigned OpName) const {
2514 const MachineOperand *Mods = getNamedOperand(MI, OpName);
2515 return Mods && Mods->getImm();
2518 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
2519 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2520 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2521 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2522 hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2523 hasModifiersSet(MI, AMDGPU::OpName::omod);
2526 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
2527 const MachineOperand &MO,
2528 const MCOperandInfo &OpInfo) const {
2529 // Literal constants use the constant bus.
2530 //if (isLiteralConstantLike(MO, OpInfo))
2533 return !isInlineConstant(MO, OpInfo);
2536 return true; // Misc other operands like FrameIndex
2541 if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
2542 return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
2544 // FLAT_SCR is just an SGPR pair.
2545 if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR))
2548 // EXEC register uses the constant bus.
2549 if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC)
2552 // SGPRs use the constant bus
2553 return (MO.getReg() == AMDGPU::VCC || MO.getReg() == AMDGPU::M0 ||
2554 (!MO.isImplicit() &&
2555 (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) ||
2556 AMDGPU::SGPR_64RegClass.contains(MO.getReg()))));
2559 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
2560 for (const MachineOperand &MO : MI.implicit_operands()) {
2561 // We only care about reads.
2565 switch (MO.getReg()) {
2568 case AMDGPU::FLAT_SCR:
2576 return AMDGPU::NoRegister;
2579 static bool shouldReadExec(const MachineInstr &MI) {
2580 if (SIInstrInfo::isVALU(MI)) {
2581 switch (MI.getOpcode()) {
2582 case AMDGPU::V_READLANE_B32:
2583 case AMDGPU::V_READLANE_B32_si:
2584 case AMDGPU::V_READLANE_B32_vi:
2585 case AMDGPU::V_WRITELANE_B32:
2586 case AMDGPU::V_WRITELANE_B32_si:
2587 case AMDGPU::V_WRITELANE_B32_vi:
2594 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
2595 SIInstrInfo::isSALU(MI) ||
2596 SIInstrInfo::isSMRD(MI))
2602 static bool isSubRegOf(const SIRegisterInfo &TRI,
2603 const MachineOperand &SuperVec,
2604 const MachineOperand &SubReg) {
2605 if (TargetRegisterInfo::isPhysicalRegister(SubReg.getReg()))
2606 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
2608 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
2609 SubReg.getReg() == SuperVec.getReg();
2612 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
2613 StringRef &ErrInfo) const {
2614 uint16_t Opcode = MI.getOpcode();
2615 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
2618 const MachineFunction *MF = MI.getParent()->getParent();
2619 const MachineRegisterInfo &MRI = MF->getRegInfo();
2621 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2622 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2623 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2625 // Make sure the number of operands is correct.
2626 const MCInstrDesc &Desc = get(Opcode);
2627 if (!Desc.isVariadic() &&
2628 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
2629 ErrInfo = "Instruction has wrong number of operands.";
2633 if (MI.isInlineAsm()) {
2634 // Verify register classes for inlineasm constraints.
2635 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
2637 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
2641 const MachineOperand &Op = MI.getOperand(I);
2645 unsigned Reg = Op.getReg();
2646 if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
2647 ErrInfo = "inlineasm operand has incorrect register class.";
2655 // Make sure the register classes are correct.
2656 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
2657 if (MI.getOperand(i).isFPImm()) {
2658 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
2659 "all fp values to integers.";
2663 int RegClass = Desc.OpInfo[i].RegClass;
2665 switch (Desc.OpInfo[i].OperandType) {
2666 case MCOI::OPERAND_REGISTER:
2667 if (MI.getOperand(i).isImm()) {
2668 ErrInfo = "Illegal immediate value for operand.";
2672 case AMDGPU::OPERAND_REG_IMM_INT32:
2673 case AMDGPU::OPERAND_REG_IMM_FP32:
2675 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2676 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2677 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2678 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2679 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2680 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
2681 const MachineOperand &MO = MI.getOperand(i);
2682 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
2683 ErrInfo = "Illegal immediate value for operand.";
2688 case MCOI::OPERAND_IMMEDIATE:
2689 case AMDGPU::OPERAND_KIMM32:
2690 // Check if this operand is an immediate.
2691 // FrameIndex operands will be replaced by immediates, so they are
2693 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
2694 ErrInfo = "Expected immediate, but got non-immediate";
2702 if (!MI.getOperand(i).isReg())
2705 if (RegClass != -1) {
2706 unsigned Reg = MI.getOperand(i).getReg();
2707 if (Reg == AMDGPU::NoRegister ||
2708 TargetRegisterInfo::isVirtualRegister(Reg))
2711 const TargetRegisterClass *RC = RI.getRegClass(RegClass);
2712 if (!RC->contains(Reg)) {
2713 ErrInfo = "Operand has incorrect register class.";
2721 if (!ST.hasSDWA()) {
2722 ErrInfo = "SDWA is not supported on this target";
2726 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2728 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
2730 for (int OpIdx: OpIndicies) {
2733 const MachineOperand &MO = MI.getOperand(OpIdx);
2735 if (!ST.hasSDWAScalar()) {
2737 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
2738 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
2742 // No immediates on GFX9
2744 ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
2750 if (!ST.hasSDWAOmod()) {
2751 // No omod allowed on VI
2752 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2753 if (OMod != nullptr &&
2754 (!OMod->isImm() || OMod->getImm() != 0)) {
2755 ErrInfo = "OMod not allowed in SDWA instructions on VI";
2760 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
2761 if (isVOPC(BasicOpcode)) {
2762 if (!ST.hasSDWASdst() && DstIdx != -1) {
2763 // Only vcc allowed as dst on VI for VOPC
2764 const MachineOperand &Dst = MI.getOperand(DstIdx);
2765 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
2766 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
2769 } else if (!ST.hasSDWAOutModsVOPC()) {
2770 // No clamp allowed on GFX9 for VOPC
2771 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2772 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
2773 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
2777 // No omod allowed on GFX9 for VOPC
2778 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2779 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
2780 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
2786 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
2787 if (DstUnused && DstUnused->isImm() &&
2788 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
2789 const MachineOperand &Dst = MI.getOperand(DstIdx);
2790 if (!Dst.isReg() || !Dst.isTied()) {
2791 ErrInfo = "Dst register should have tied register";
2795 const MachineOperand &TiedMO =
2796 MI.getOperand(MI.findTiedOperandIdx(DstIdx));
2797 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
2799 "Dst register should be tied to implicit use of preserved register";
2801 } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
2802 Dst.getReg() != TiedMO.getReg()) {
2803 ErrInfo = "Dst register should use same physical register as preserved";
2809 // Verify VOP*. Ignore multiple sgpr operands on writelane.
2810 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
2811 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
2812 // Only look at the true operands. Only a real operand can use the constant
2813 // bus, and we don't want to check pseudo-operands like the source modifier
2815 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2817 unsigned ConstantBusCount = 0;
2818 unsigned LiteralCount = 0;
2820 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
2823 unsigned SGPRUsed = findImplicitSGPRRead(MI);
2824 if (SGPRUsed != AMDGPU::NoRegister)
2827 for (int OpIdx : OpIndices) {
2830 const MachineOperand &MO = MI.getOperand(OpIdx);
2831 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
2833 if (MO.getReg() != SGPRUsed)
2835 SGPRUsed = MO.getReg();
2842 if (ConstantBusCount > 1) {
2843 ErrInfo = "VOP* instruction uses the constant bus more than once";
2847 if (isVOP3(MI) && LiteralCount) {
2848 ErrInfo = "VOP3 instruction uses literal";
2853 // Verify misc. restrictions on specific instructions.
2854 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
2855 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
2856 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
2857 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
2858 const MachineOperand &Src2 = MI.getOperand(Src2Idx);
2859 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
2860 if (!compareMachineOp(Src0, Src1) &&
2861 !compareMachineOp(Src0, Src2)) {
2862 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
2869 int64_t Imm = getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
2870 if (sopkIsZext(MI)) {
2871 if (!isUInt<16>(Imm)) {
2872 ErrInfo = "invalid immediate for SOPK instruction";
2876 if (!isInt<16>(Imm)) {
2877 ErrInfo = "invalid immediate for SOPK instruction";
2883 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
2884 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
2885 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
2886 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
2887 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
2888 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
2890 const unsigned StaticNumOps = Desc.getNumOperands() +
2891 Desc.getNumImplicitUses();
2892 const unsigned NumImplicitOps = IsDst ? 2 : 1;
2894 // Allow additional implicit operands. This allows a fixup done by the post
2895 // RA scheduler where the main implicit operand is killed and implicit-defs
2896 // are added for sub-registers that remain live after this instruction.
2897 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
2898 ErrInfo = "missing implicit register operands";
2902 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2904 if (!Dst->isUse()) {
2905 ErrInfo = "v_movreld_b32 vdst should be a use operand";
2910 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
2911 UseOpIdx != StaticNumOps + 1) {
2912 ErrInfo = "movrel implicit operands should be tied";
2917 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
2918 const MachineOperand &ImpUse
2919 = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
2920 if (!ImpUse.isReg() || !ImpUse.isUse() ||
2921 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
2922 ErrInfo = "src0 should be subreg of implicit vector use";
2927 // Make sure we aren't losing exec uses in the td files. This mostly requires
2928 // being careful when using let Uses to try to add other use registers.
2929 if (shouldReadExec(MI)) {
2930 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
2931 ErrInfo = "VALU instruction does not implicitly read exec mask";
2937 if (MI.mayStore()) {
2938 // The register offset form of scalar stores may only use m0 as the
2939 // soffset register.
2940 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
2941 if (Soff && Soff->getReg() != AMDGPU::M0) {
2942 ErrInfo = "scalar stores must use m0 as offset register";
2948 if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
2949 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
2950 if (Offset->getImm() != 0) {
2951 ErrInfo = "subtarget does not support offsets in flat instructions";
2956 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
2958 using namespace AMDGPU::DPP;
2960 unsigned DC = DppCt->getImm();
2961 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
2962 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
2963 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
2964 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
2965 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
2966 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST)) {
2967 ErrInfo = "Invalid dpp_ctrl value";
2975 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
2976 switch (MI.getOpcode()) {
2977 default: return AMDGPU::INSTRUCTION_LIST_END;
2978 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
2979 case AMDGPU::COPY: return AMDGPU::COPY;
2980 case AMDGPU::PHI: return AMDGPU::PHI;
2981 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
2982 case AMDGPU::WQM: return AMDGPU::WQM;
2983 case AMDGPU::WWM: return AMDGPU::WWM;
2984 case AMDGPU::S_MOV_B32:
2985 return MI.getOperand(1).isReg() ?
2986 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
2987 case AMDGPU::S_ADD_I32:
2988 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
2989 case AMDGPU::S_ADDC_U32:
2990 return AMDGPU::V_ADDC_U32_e32;
2991 case AMDGPU::S_SUB_I32:
2992 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
2993 // FIXME: These are not consistently handled, and selected when the carry is
2995 case AMDGPU::S_ADD_U32:
2996 return AMDGPU::V_ADD_I32_e32;
2997 case AMDGPU::S_SUB_U32:
2998 return AMDGPU::V_SUB_I32_e32;
2999 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3000 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32;
3001 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3002 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3003 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3004 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3005 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3006 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3007 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3008 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3009 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3010 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3011 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3012 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3013 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3014 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3015 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3016 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3017 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3018 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3019 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3020 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3021 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3022 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3023 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3024 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3025 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3026 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3027 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3028 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3029 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3030 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3031 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3032 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3033 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3034 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3035 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3036 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3037 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3038 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3039 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3040 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3041 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3045 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
3046 unsigned OpNo) const {
3047 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3048 const MCInstrDesc &Desc = get(MI.getOpcode());
3049 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3050 Desc.OpInfo[OpNo].RegClass == -1) {
3051 unsigned Reg = MI.getOperand(OpNo).getReg();
3053 if (TargetRegisterInfo::isVirtualRegister(Reg))
3054 return MRI.getRegClass(Reg);
3055 return RI.getPhysRegClass(Reg);
3058 unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3059 return RI.getRegClass(RCID);
3062 bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const {
3063 switch (MI.getOpcode()) {
3065 case AMDGPU::REG_SEQUENCE:
3067 case AMDGPU::INSERT_SUBREG:
3068 return RI.hasVGPRs(getOpRegClass(MI, 0));
3070 return RI.hasVGPRs(getOpRegClass(MI, OpNo));
3074 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3075 MachineBasicBlock::iterator I = MI;
3076 MachineBasicBlock *MBB = MI.getParent();
3077 MachineOperand &MO = MI.getOperand(OpIdx);
3078 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
3079 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3080 const TargetRegisterClass *RC = RI.getRegClass(RCID);
3081 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
3083 Opcode = AMDGPU::COPY;
3084 else if (RI.isSGPRClass(RC))
3085 Opcode = AMDGPU::S_MOV_B32;
3087 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3088 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3089 VRC = &AMDGPU::VReg_64RegClass;
3091 VRC = &AMDGPU::VGPR_32RegClass;
3093 unsigned Reg = MRI.createVirtualRegister(VRC);
3094 DebugLoc DL = MBB->findDebugLoc(I);
3095 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3096 MO.ChangeToRegister(Reg, false);
3099 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI,
3100 MachineRegisterInfo &MRI,
3101 MachineOperand &SuperReg,
3102 const TargetRegisterClass *SuperRC,
3104 const TargetRegisterClass *SubRC)
3106 MachineBasicBlock *MBB = MI->getParent();
3107 DebugLoc DL = MI->getDebugLoc();
3108 unsigned SubReg = MRI.createVirtualRegister(SubRC);
3110 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3111 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3112 .addReg(SuperReg.getReg(), 0, SubIdx);
3116 // Just in case the super register is itself a sub-register, copy it to a new
3117 // value so we don't need to worry about merging its subreg index with the
3118 // SubIdx passed to this function. The register coalescer should be able to
3119 // eliminate this extra copy.
3120 unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3122 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3123 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3125 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3126 .addReg(NewSuperReg, 0, SubIdx);
3131 MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
3132 MachineBasicBlock::iterator MII,
3133 MachineRegisterInfo &MRI,
3135 const TargetRegisterClass *SuperRC,
3137 const TargetRegisterClass *SubRC) const {
3139 if (SubIdx == AMDGPU::sub0)
3140 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3141 if (SubIdx == AMDGPU::sub1)
3142 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3144 llvm_unreachable("Unhandled register index for immediate");
3147 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3149 return MachineOperand::CreateReg(SubReg, false);
3152 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3153 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3154 assert(Inst.getNumExplicitOperands() == 3);
3155 MachineOperand Op1 = Inst.getOperand(1);
3156 Inst.RemoveOperand(1);
3157 Inst.addOperand(Op1);
3160 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
3161 const MCOperandInfo &OpInfo,
3162 const MachineOperand &MO) const {
3166 unsigned Reg = MO.getReg();
3167 const TargetRegisterClass *RC =
3168 TargetRegisterInfo::isVirtualRegister(Reg) ?
3169 MRI.getRegClass(Reg) :
3170 RI.getPhysRegClass(Reg);
3172 const SIRegisterInfo *TRI =
3173 static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3174 RC = TRI->getSubRegClass(RC, MO.getSubReg());
3176 // In order to be legal, the common sub-class must be equal to the
3177 // class of the current operand. For example:
3179 // v_mov_b32 s0 ; Operand defined as vsrc_b32
3180 // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3182 // s_sendmsg 0, s0 ; Operand defined as m0reg
3183 // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3185 return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3188 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
3189 const MCOperandInfo &OpInfo,
3190 const MachineOperand &MO) const {
3192 return isLegalRegOperand(MRI, OpInfo, MO);
3194 // Handle non-register types that are treated like immediates.
3195 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
3199 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3200 const MachineOperand *MO) const {
3201 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3202 const MCInstrDesc &InstDesc = MI.getDesc();
3203 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3204 const TargetRegisterClass *DefinedRC =
3205 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3207 MO = &MI.getOperand(OpIdx);
3209 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3211 RegSubRegPair SGPRUsed;
3213 SGPRUsed = RegSubRegPair(MO->getReg(), MO->getSubReg());
3215 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3218 const MachineOperand &Op = MI.getOperand(i);
3220 if ((Op.getReg() != SGPRUsed.Reg || Op.getSubReg() != SGPRUsed.SubReg) &&
3221 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3224 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3232 return isLegalRegOperand(MRI, OpInfo, *MO);
3235 // Handle non-register types that are treated like immediates.
3236 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI());
3239 // This operand expects an immediate.
3243 return isImmOperandLegal(MI, OpIdx, *MO);
3246 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
3247 MachineInstr &MI) const {
3248 unsigned Opc = MI.getOpcode();
3249 const MCInstrDesc &InstrDesc = get(Opc);
3251 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3252 MachineOperand &Src1 = MI.getOperand(Src1Idx);
3254 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3255 // we need to only have one constant bus use.
3257 // Note we do not need to worry about literal constants here. They are
3258 // disabled for the operand type for instructions because they will always
3259 // violate the one constant bus use rule.
3260 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3261 if (HasImplicitSGPR) {
3262 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3263 MachineOperand &Src0 = MI.getOperand(Src0Idx);
3265 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg()))
3266 legalizeOpWithMove(MI, Src0Idx);
3269 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3270 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3271 // src0/src1 with V_READFIRSTLANE.
3272 if (Opc == AMDGPU::V_WRITELANE_B32) {
3273 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3274 MachineOperand &Src0 = MI.getOperand(Src0Idx);
3275 const DebugLoc &DL = MI.getDebugLoc();
3276 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3277 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3278 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3280 Src0.ChangeToRegister(Reg, false);
3282 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3283 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3284 const DebugLoc &DL = MI.getDebugLoc();
3285 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3287 Src1.ChangeToRegister(Reg, false);
3292 // VOP2 src0 instructions support all operand types, so we don't need to check
3293 // their legality. If src1 is already legal, we don't need to do anything.
3294 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3297 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3298 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3299 // select is uniform.
3300 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3301 RI.isVGPR(MRI, Src1.getReg())) {
3302 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3303 const DebugLoc &DL = MI.getDebugLoc();
3304 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3306 Src1.ChangeToRegister(Reg, false);
3310 // We do not use commuteInstruction here because it is too aggressive and will
3311 // commute if it is possible. We only want to commute here if it improves
3312 // legality. This can be called a fairly large number of times so don't waste
3313 // compile time pointlessly swapping and checking legality again.
3314 if (HasImplicitSGPR || !MI.isCommutable()) {
3315 legalizeOpWithMove(MI, Src1Idx);
3319 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3320 MachineOperand &Src0 = MI.getOperand(Src0Idx);
3322 // If src0 can be used as src1, commuting will make the operands legal.
3323 // Otherwise we have to give up and insert a move.
3325 // TODO: Other immediate-like operand kinds could be commuted if there was a
3326 // MachineOperand::ChangeTo* for them.
3327 if ((!Src1.isImm() && !Src1.isReg()) ||
3328 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3329 legalizeOpWithMove(MI, Src1Idx);
3333 int CommutedOpc = commuteOpcode(MI);
3334 if (CommutedOpc == -1) {
3335 legalizeOpWithMove(MI, Src1Idx);
3339 MI.setDesc(get(CommutedOpc));
3341 unsigned Src0Reg = Src0.getReg();
3342 unsigned Src0SubReg = Src0.getSubReg();
3343 bool Src0Kill = Src0.isKill();
3346 Src0.ChangeToImmediate(Src1.getImm());
3347 else if (Src1.isReg()) {
3348 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
3349 Src0.setSubReg(Src1.getSubReg());
3351 llvm_unreachable("Should only have register or immediate operands");
3353 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
3354 Src1.setSubReg(Src0SubReg);
3357 // Legalize VOP3 operands. Because all operand types are supported for any
3358 // operand, and since literal constants are not allowed and should never be
3359 // seen, we only need to worry about inserting copies if we use multiple SGPR
3361 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
3362 MachineInstr &MI) const {
3363 unsigned Opc = MI.getOpcode();
3366 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
3367 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
3368 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
3371 // Find the one SGPR operand we are allowed to use.
3372 unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
3374 for (unsigned i = 0; i < 3; ++i) {
3375 int Idx = VOP3Idx[i];
3378 MachineOperand &MO = MI.getOperand(Idx);
3380 // We should never see a VOP3 instruction with an illegal immediate operand.
3384 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
3385 continue; // VGPRs are legal
3387 if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) {
3388 SGPRReg = MO.getReg();
3389 // We can use one SGPR in each VOP3 instruction.
3393 // If we make it this far, then the operand is not legal and we must
3395 legalizeOpWithMove(MI, Idx);
3399 unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
3400 MachineRegisterInfo &MRI) const {
3401 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
3402 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
3403 unsigned DstReg = MRI.createVirtualRegister(SRC);
3404 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
3407 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3408 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
3413 SmallVector<unsigned, 8> SRegs;
3414 for (unsigned i = 0; i < SubRegs; ++i) {
3415 unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3416 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3417 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
3418 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
3419 SRegs.push_back(SGPR);
3422 MachineInstrBuilder MIB =
3423 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3424 get(AMDGPU::REG_SEQUENCE), DstReg);
3425 for (unsigned i = 0; i < SubRegs; ++i) {
3426 MIB.addReg(SRegs[i]);
3427 MIB.addImm(RI.getSubRegFromChannel(i));
3432 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
3433 MachineInstr &MI) const {
3435 // If the pointer is store in VGPRs, then we need to move them to
3436 // SGPRs using v_readfirstlane. This is safe because we only select
3437 // loads with uniform pointers to SMRD instruction so we know the
3438 // pointer value is uniform.
3439 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
3440 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
3441 unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
3442 SBase->setReg(SGPR);
3446 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
3447 MachineBasicBlock::iterator I,
3448 const TargetRegisterClass *DstRC,
3450 MachineRegisterInfo &MRI,
3451 const DebugLoc &DL) const {
3452 unsigned OpReg = Op.getReg();
3453 unsigned OpSubReg = Op.getSubReg();
3455 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
3456 RI.getRegClassForReg(MRI, OpReg), OpSubReg);
3458 // Check if operand is already the correct register class.
3462 unsigned DstReg = MRI.createVirtualRegister(DstRC);
3463 MachineInstr *Copy =
3464 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
3469 MachineInstr *Def = MRI.getVRegDef(OpReg);
3473 // Try to eliminate the copy if it is copying an immediate value.
3474 if (Def->isMoveImmediate())
3475 FoldImmediate(*Copy, *Def, OpReg, &MRI);
3478 void SIInstrInfo::legalizeOperands(MachineInstr &MI) const {
3479 MachineFunction &MF = *MI.getParent()->getParent();
3480 MachineRegisterInfo &MRI = MF.getRegInfo();
3483 if (isVOP2(MI) || isVOPC(MI)) {
3484 legalizeOperandsVOP2(MRI, MI);
3490 legalizeOperandsVOP3(MRI, MI);
3496 legalizeOperandsSMRD(MRI, MI);
3500 // Legalize REG_SEQUENCE and PHI
3501 // The register class of the operands much be the same type as the register
3502 // class of the output.
3503 if (MI.getOpcode() == AMDGPU::PHI) {
3504 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
3505 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3506 if (!MI.getOperand(i).isReg() ||
3507 !TargetRegisterInfo::isVirtualRegister(MI.getOperand(i).getReg()))
3509 const TargetRegisterClass *OpRC =
3510 MRI.getRegClass(MI.getOperand(i).getReg());
3511 if (RI.hasVGPRs(OpRC)) {
3518 // If any of the operands are VGPR registers, then they all most be
3519 // otherwise we will create illegal VGPR->SGPR copies when legalizing
3521 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
3524 VRC = RI.getEquivalentVGPRClass(SRC);
3531 // Update all the operands so they have the same type.
3532 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3533 MachineOperand &Op = MI.getOperand(I);
3534 if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg()))
3537 // MI is a PHI instruction.
3538 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
3539 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
3541 // Avoid creating no-op copies with the same src and dst reg class. These
3542 // confuse some of the machine passes.
3543 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
3547 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
3548 // VGPR dest type and SGPR sources, insert copies so all operands are
3549 // VGPRs. This seems to help operand folding / the register coalescer.
3550 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
3551 MachineBasicBlock *MBB = MI.getParent();
3552 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
3553 if (RI.hasVGPRs(DstRC)) {
3554 // Update all the operands so they are VGPR register classes. These may
3555 // not be the same register class because REG_SEQUENCE supports mixing
3556 // subregister index types e.g. sub0_sub1 + sub2 + sub3
3557 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3558 MachineOperand &Op = MI.getOperand(I);
3559 if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg()))
3562 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
3563 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
3567 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
3575 // Legalize INSERT_SUBREG
3576 // src0 must have the same register class as dst
3577 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
3578 unsigned Dst = MI.getOperand(0).getReg();
3579 unsigned Src0 = MI.getOperand(1).getReg();
3580 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
3581 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
3582 if (DstRC != Src0RC) {
3583 MachineBasicBlock *MBB = MI.getParent();
3584 MachineOperand &Op = MI.getOperand(1);
3585 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
3590 // Legalize SI_INIT_M0
3591 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
3592 MachineOperand &Src = MI.getOperand(0);
3593 if (Src.isReg() && RI.hasVGPRs(MRI.getRegClass(Src.getReg())))
3594 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
3598 // Legalize MIMG and MUBUF/MTBUF for shaders.
3600 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
3601 // scratch memory access. In both cases, the legalization never involves
3602 // conversion to the addr64 form.
3604 (AMDGPU::isShader(MF.getFunction().getCallingConv()) &&
3605 (isMUBUF(MI) || isMTBUF(MI)))) {
3606 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
3607 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
3608 unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
3609 SRsrc->setReg(SGPR);
3612 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
3613 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
3614 unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
3615 SSamp->setReg(SGPR);
3620 // Legalize MUBUF* instructions by converting to addr64 form.
3621 // FIXME: If we start using the non-addr64 instructions for compute, we
3622 // may need to legalize them as above. This especially applies to the
3623 // buffer_load_format_* variants and variants with idxen (or bothen).
3625 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
3626 if (SRsrcIdx != -1) {
3627 // We have an MUBUF instruction
3628 MachineOperand *SRsrc = &MI.getOperand(SRsrcIdx);
3629 unsigned SRsrcRC = get(MI.getOpcode()).OpInfo[SRsrcIdx].RegClass;
3630 if (RI.getCommonSubClass(MRI.getRegClass(SRsrc->getReg()),
3631 RI.getRegClass(SRsrcRC))) {
3632 // The operands are legal.
3633 // FIXME: We may need to legalize operands besided srsrc.
3637 MachineBasicBlock &MBB = *MI.getParent();
3639 // Extract the ptr from the resource descriptor.
3640 unsigned SRsrcPtr = buildExtractSubReg(MI, MRI, *SRsrc,
3641 &AMDGPU::VReg_128RegClass, AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
3643 // Create an empty resource descriptor
3644 unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3645 unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3646 unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3647 unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3648 uint64_t RsrcDataFormat = getDefaultRsrcDataFormat();
3651 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::S_MOV_B64), Zero64)
3654 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
3655 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
3656 .addImm(RsrcDataFormat & 0xFFFFFFFF);
3658 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
3659 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
3660 .addImm(RsrcDataFormat >> 32);
3662 // NewSRsrc = {Zero64, SRsrcFormat}
3663 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewSRsrc)
3665 .addImm(AMDGPU::sub0_sub1)
3666 .addReg(SRsrcFormatLo)
3667 .addImm(AMDGPU::sub2)
3668 .addReg(SRsrcFormatHi)
3669 .addImm(AMDGPU::sub3);
3671 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
3672 unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
3674 // This is already an ADDR64 instruction so we need to add the pointer
3675 // extracted from the resource descriptor to the current value of VAddr.
3676 unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3677 unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3679 // NewVaddrLo = SRsrcPtr:sub0 + VAddr:sub0
3680 DebugLoc DL = MI.getDebugLoc();
3681 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo)
3682 .addReg(SRsrcPtr, 0, AMDGPU::sub0)
3683 .addReg(VAddr->getReg(), 0, AMDGPU::sub0);
3685 // NewVaddrHi = SRsrcPtr:sub1 + VAddr:sub1
3686 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi)
3687 .addReg(SRsrcPtr, 0, AMDGPU::sub1)
3688 .addReg(VAddr->getReg(), 0, AMDGPU::sub1);
3690 // NewVaddr = {NewVaddrHi, NewVaddrLo}
3691 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
3693 .addImm(AMDGPU::sub0)
3695 .addImm(AMDGPU::sub1);
3697 // This instructions is the _OFFSET variant, so we need to convert it to
3699 assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
3700 < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
3701 "FIXME: Need to emit flat atomics here");
3703 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
3704 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3705 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
3706 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
3708 // Atomics rith return have have an additional tied operand and are
3709 // missing some of the special bits.
3710 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
3711 MachineInstr *Addr64;
3714 // Regular buffer load / store.
3715 MachineInstrBuilder MIB =
3716 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
3718 .addReg(AMDGPU::NoRegister) // Dummy value for vaddr.
3719 // This will be replaced later
3720 // with the new value of vaddr.
3725 // Atomics do not have this operand.
3726 if (const MachineOperand *GLC =
3727 getNamedOperand(MI, AMDGPU::OpName::glc)) {
3728 MIB.addImm(GLC->getImm());
3731 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
3733 if (const MachineOperand *TFE =
3734 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
3735 MIB.addImm(TFE->getImm());
3738 MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
3741 // Atomics with return.
3742 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
3745 .addReg(AMDGPU::NoRegister) // Dummy value for vaddr.
3746 // This will be replaced later
3747 // with the new value of vaddr.
3751 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
3752 .setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
3755 MI.removeFromParent();
3757 // NewVaddr = {NewVaddrHi, NewVaddrLo}
3758 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
3760 .addReg(SRsrcPtr, 0, AMDGPU::sub0)
3761 .addImm(AMDGPU::sub0)
3762 .addReg(SRsrcPtr, 0, AMDGPU::sub1)
3763 .addImm(AMDGPU::sub1);
3765 VAddr = getNamedOperand(*Addr64, AMDGPU::OpName::vaddr);
3766 SRsrc = getNamedOperand(*Addr64, AMDGPU::OpName::srsrc);
3769 // Update the instruction to use NewVaddr
3770 VAddr->setReg(NewVAddr);
3771 // Update the instruction to use NewSRsrc
3772 SRsrc->setReg(NewSRsrc);
3776 void SIInstrInfo::moveToVALU(MachineInstr &TopInst) const {
3777 SetVectorType Worklist;
3778 Worklist.insert(&TopInst);
3780 while (!Worklist.empty()) {
3781 MachineInstr &Inst = *Worklist.pop_back_val();
3782 MachineBasicBlock *MBB = Inst.getParent();
3783 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
3785 unsigned Opcode = Inst.getOpcode();
3786 unsigned NewOpcode = getVALUOp(Inst);
3788 // Handle some special cases
3792 case AMDGPU::S_ADD_U64_PSEUDO:
3793 case AMDGPU::S_SUB_U64_PSEUDO:
3794 splitScalar64BitAddSub(Worklist, Inst);
3795 Inst.eraseFromParent();
3797 case AMDGPU::S_ADD_I32:
3798 case AMDGPU::S_SUB_I32:
3799 // FIXME: The u32 versions currently selected use the carry.
3800 if (moveScalarAddSub(Worklist, Inst))
3805 case AMDGPU::S_AND_B64:
3806 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_AND_B32_e64);
3807 Inst.eraseFromParent();
3810 case AMDGPU::S_OR_B64:
3811 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_OR_B32_e64);
3812 Inst.eraseFromParent();
3815 case AMDGPU::S_XOR_B64:
3816 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_XOR_B32_e64);
3817 Inst.eraseFromParent();
3820 case AMDGPU::S_NOT_B64:
3821 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::V_NOT_B32_e32);
3822 Inst.eraseFromParent();
3825 case AMDGPU::S_BCNT1_I32_B64:
3826 splitScalar64BitBCNT(Worklist, Inst);
3827 Inst.eraseFromParent();
3830 case AMDGPU::S_BFE_I64:
3831 splitScalar64BitBFE(Worklist, Inst);
3832 Inst.eraseFromParent();
3835 case AMDGPU::S_LSHL_B32:
3836 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
3837 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
3841 case AMDGPU::S_ASHR_I32:
3842 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
3843 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
3847 case AMDGPU::S_LSHR_B32:
3848 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
3849 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
3853 case AMDGPU::S_LSHL_B64:
3854 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
3855 NewOpcode = AMDGPU::V_LSHLREV_B64;
3859 case AMDGPU::S_ASHR_I64:
3860 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
3861 NewOpcode = AMDGPU::V_ASHRREV_I64;
3865 case AMDGPU::S_LSHR_B64:
3866 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
3867 NewOpcode = AMDGPU::V_LSHRREV_B64;
3872 case AMDGPU::S_ABS_I32:
3873 lowerScalarAbs(Worklist, Inst);
3874 Inst.eraseFromParent();
3877 case AMDGPU::S_CBRANCH_SCC0:
3878 case AMDGPU::S_CBRANCH_SCC1:
3879 // Clear unused bits of vcc
3880 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
3882 .addReg(AMDGPU::EXEC)
3883 .addReg(AMDGPU::VCC);
3886 case AMDGPU::S_BFE_U64:
3887 case AMDGPU::S_BFM_B64:
3888 llvm_unreachable("Moving this op to VALU not implemented");
3890 case AMDGPU::S_PACK_LL_B32_B16:
3891 case AMDGPU::S_PACK_LH_B32_B16:
3892 case AMDGPU::S_PACK_HH_B32_B16:
3893 movePackToVALU(Worklist, MRI, Inst);
3894 Inst.eraseFromParent();
3897 case AMDGPU::S_XNOR_B32:
3898 lowerScalarXnor(Worklist, Inst);
3899 Inst.eraseFromParent();
3902 case AMDGPU::S_XNOR_B64:
3903 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32);
3904 Inst.eraseFromParent();
3907 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR: {
3908 unsigned VDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3909 const MachineOperand *VAddr = getNamedOperand(Inst, AMDGPU::OpName::soff);
3910 auto Add = MRI.getUniqueVRegDef(VAddr->getReg());
3911 unsigned Offset = 0;
3913 // FIXME: This isn't safe because the addressing mode doesn't work
3914 // correctly if vaddr is negative.
3916 // FIXME: Should probably be done somewhere else, maybe SIFoldOperands.
3918 // See if we can extract an immediate offset by recognizing one of these:
3919 // V_ADD_I32_e32 dst, imm, src1
3920 // V_ADD_I32_e32 dst, (S_MOV_B32 imm), src1
3921 // V_ADD will be removed by "Remove dead machine instructions".
3923 (Add->getOpcode() == AMDGPU::V_ADD_I32_e32 ||
3924 Add->getOpcode() == AMDGPU::V_ADD_U32_e64)) {
3925 static const unsigned SrcNames[2] = {
3926 AMDGPU::OpName::src0,
3927 AMDGPU::OpName::src1,
3930 // Find a literal offset in one of source operands.
3931 for (int i = 0; i < 2; i++) {
3932 const MachineOperand *Src =
3933 getNamedOperand(*Add, SrcNames[i]);
3936 auto Mov = MRI.getUniqueVRegDef(Src->getReg());
3937 if (Mov && Mov->getOpcode() == AMDGPU::S_MOV_B32)
3938 Src = &Mov->getOperand(1);
3943 Offset = Src->getImm();
3944 else if (Src->isCImm())
3945 Offset = Src->getCImm()->getZExtValue();
3948 if (Offset && isLegalMUBUFImmOffset(Offset)) {
3949 VAddr = getNamedOperand(*Add, SrcNames[!i]);
3957 MachineInstr *NewInstr =
3958 BuildMI(*MBB, Inst, Inst.getDebugLoc(),
3959 get(AMDGPU::BUFFER_LOAD_DWORD_OFFEN), VDst)
3960 .add(*VAddr) // vaddr
3961 .add(*getNamedOperand(Inst, AMDGPU::OpName::sbase)) // srsrc
3962 .addImm(0) // soffset
3963 .addImm(Offset) // offset
3964 .addImm(getNamedOperand(Inst, AMDGPU::OpName::glc)->getImm())
3967 .setMemRefs(Inst.memoperands_begin(), Inst.memoperands_end())
3970 MRI.replaceRegWith(getNamedOperand(Inst, AMDGPU::OpName::sdst)->getReg(),
3972 addUsersToMoveToVALUWorklist(VDst, MRI, Worklist);
3973 Inst.eraseFromParent();
3975 // Legalize all operands other than the offset. Notably, convert the srsrc
3976 // into SGPRs using v_readfirstlane if needed.
3977 legalizeOperands(*NewInstr);
3982 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
3983 // We cannot move this instruction to the VALU, so we should try to
3984 // legalize its operands instead.
3985 legalizeOperands(Inst);
3989 // Use the new VALU Opcode.
3990 const MCInstrDesc &NewDesc = get(NewOpcode);
3991 Inst.setDesc(NewDesc);
3993 // Remove any references to SCC. Vector instructions can't read from it, and
3994 // We're just about to add the implicit use / defs of VCC, and we don't want
3996 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
3997 MachineOperand &Op = Inst.getOperand(i);
3998 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
3999 Inst.RemoveOperand(i);
4000 addSCCDefUsersToVALUWorklist(Inst, Worklist);
4004 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4005 // We are converting these to a BFE, so we need to add the missing
4006 // operands for the size and offset.
4007 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4008 Inst.addOperand(MachineOperand::CreateImm(0));
4009 Inst.addOperand(MachineOperand::CreateImm(Size));
4011 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4012 // The VALU version adds the second operand to the result, so insert an
4014 Inst.addOperand(MachineOperand::CreateImm(0));
4017 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent());
4019 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4020 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4021 // If we need to move this to VGPRs, we need to unpack the second operand
4022 // back into the 2 separate ones for bit offset and width.
4023 assert(OffsetWidthOp.isImm() &&
4024 "Scalar BFE is only implemented for constant width and offset");
4025 uint32_t Imm = OffsetWidthOp.getImm();
4027 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4028 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4029 Inst.RemoveOperand(2); // Remove old immediate.
4030 Inst.addOperand(MachineOperand::CreateImm(Offset));
4031 Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4034 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4035 unsigned NewDstReg = AMDGPU::NoRegister;
4037 unsigned DstReg = Inst.getOperand(0).getReg();
4038 if (TargetRegisterInfo::isPhysicalRegister(DstReg))
4041 // Update the destination register class.
4042 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4046 if (Inst.isCopy() &&
4047 TargetRegisterInfo::isVirtualRegister(Inst.getOperand(1).getReg()) &&
4048 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4049 // Instead of creating a copy where src and dst are the same register
4050 // class, we just replace all uses of dst with src. These kinds of
4051 // copies interfere with the heuristics MachineSink uses to decide
4052 // whether or not to split a critical edge. Since the pass assumes
4053 // that copies will end up as machine instructions and not be
4055 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4056 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4057 MRI.clearKillFlags(Inst.getOperand(1).getReg());
4058 Inst.getOperand(0).setReg(DstReg);
4060 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4061 // these are deleted later, but at -O0 it would leave a suspicious
4062 // looking illegal copy of an undef register.
4063 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4064 Inst.RemoveOperand(I);
4065 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4069 NewDstReg = MRI.createVirtualRegister(NewDstRC);
4070 MRI.replaceRegWith(DstReg, NewDstReg);
4073 // Legalize the operands
4074 legalizeOperands(Inst);
4077 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4081 // Add/sub require special handling to deal with carry outs.
4082 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist,
4083 MachineInstr &Inst) const {
4084 if (ST.hasAddNoCarry()) {
4085 // Assume there is no user of scc since we don't select this in that case.
4086 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4089 MachineBasicBlock &MBB = *Inst.getParent();
4090 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4092 unsigned OldDstReg = Inst.getOperand(0).getReg();
4093 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4095 unsigned Opc = Inst.getOpcode();
4096 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4098 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4099 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
4101 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
4102 Inst.RemoveOperand(3);
4104 Inst.setDesc(get(NewOpc));
4105 Inst.addImplicitDefUseOperands(*MBB.getParent());
4106 MRI.replaceRegWith(OldDstReg, ResultReg);
4107 legalizeOperands(Inst);
4109 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4116 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
4117 MachineInstr &Inst) const {
4118 MachineBasicBlock &MBB = *Inst.getParent();
4119 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4120 MachineBasicBlock::iterator MII = Inst;
4121 DebugLoc DL = Inst.getDebugLoc();
4123 MachineOperand &Dest = Inst.getOperand(0);
4124 MachineOperand &Src = Inst.getOperand(1);
4125 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4126 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4128 unsigned SubOp = ST.hasAddNoCarry() ?
4129 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
4131 BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
4133 .addReg(Src.getReg());
4135 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
4136 .addReg(Src.getReg())
4139 MRI.replaceRegWith(Dest.getReg(), ResultReg);
4140 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4143 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
4144 MachineInstr &Inst) const {
4145 MachineBasicBlock &MBB = *Inst.getParent();
4146 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4147 MachineBasicBlock::iterator MII = Inst;
4148 const DebugLoc &DL = Inst.getDebugLoc();
4150 MachineOperand &Dest = Inst.getOperand(0);
4151 MachineOperand &Src0 = Inst.getOperand(1);
4152 MachineOperand &Src1 = Inst.getOperand(2);
4154 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
4155 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
4157 unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4158 if (ST.hasDLInsts()) {
4159 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
4163 unsigned Xor = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4164 BuildMI(MBB, MII, DL, get(AMDGPU::V_XOR_B32_e64), Xor)
4168 BuildMI(MBB, MII, DL, get(AMDGPU::V_NOT_B32_e64), NewDest)
4172 MRI.replaceRegWith(Dest.getReg(), NewDest);
4173 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4176 void SIInstrInfo::splitScalar64BitUnaryOp(
4177 SetVectorType &Worklist, MachineInstr &Inst,
4178 unsigned Opcode) const {
4179 MachineBasicBlock &MBB = *Inst.getParent();
4180 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4182 MachineOperand &Dest = Inst.getOperand(0);
4183 MachineOperand &Src0 = Inst.getOperand(1);
4184 DebugLoc DL = Inst.getDebugLoc();
4186 MachineBasicBlock::iterator MII = Inst;
4188 const MCInstrDesc &InstDesc = get(Opcode);
4189 const TargetRegisterClass *Src0RC = Src0.isReg() ?
4190 MRI.getRegClass(Src0.getReg()) :
4191 &AMDGPU::SGPR_32RegClass;
4193 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4195 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4196 AMDGPU::sub0, Src0SubRC);
4198 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4199 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4200 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4202 unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4203 BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
4205 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4206 AMDGPU::sub1, Src0SubRC);
4208 unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4209 BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
4211 unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4212 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4214 .addImm(AMDGPU::sub0)
4216 .addImm(AMDGPU::sub1);
4218 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4220 // We don't need to legalizeOperands here because for a single operand, src0
4221 // will support any kind of input.
4223 // Move all users of this moved value.
4224 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4227 void SIInstrInfo::splitScalar64BitAddSub(
4228 SetVectorType &Worklist, MachineInstr &Inst) const {
4229 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
4231 MachineBasicBlock &MBB = *Inst.getParent();
4232 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4234 unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4235 unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4236 unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4238 unsigned CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4239 unsigned DeadCarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4241 MachineOperand &Dest = Inst.getOperand(0);
4242 MachineOperand &Src0 = Inst.getOperand(1);
4243 MachineOperand &Src1 = Inst.getOperand(2);
4244 const DebugLoc &DL = Inst.getDebugLoc();
4245 MachineBasicBlock::iterator MII = Inst;
4247 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
4248 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
4249 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4250 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4252 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4253 AMDGPU::sub0, Src0SubRC);
4254 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4255 AMDGPU::sub0, Src1SubRC);
4258 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4259 AMDGPU::sub1, Src0SubRC);
4260 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4261 AMDGPU::sub1, Src1SubRC);
4263 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
4264 MachineInstr *LoHalf =
4265 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
4266 .addReg(CarryReg, RegState::Define)
4270 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
4271 MachineInstr *HiHalf =
4272 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
4273 .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
4276 .addReg(CarryReg, RegState::Kill);
4278 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4280 .addImm(AMDGPU::sub0)
4282 .addImm(AMDGPU::sub1);
4284 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4286 // Try to legalize the operands in case we need to swap the order to keep it
4288 legalizeOperands(*LoHalf);
4289 legalizeOperands(*HiHalf);
4291 // Move all users of this moved vlaue.
4292 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4295 void SIInstrInfo::splitScalar64BitBinaryOp(
4296 SetVectorType &Worklist, MachineInstr &Inst,
4297 unsigned Opcode) const {
4298 MachineBasicBlock &MBB = *Inst.getParent();
4299 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4301 MachineOperand &Dest = Inst.getOperand(0);
4302 MachineOperand &Src0 = Inst.getOperand(1);
4303 MachineOperand &Src1 = Inst.getOperand(2);
4304 DebugLoc DL = Inst.getDebugLoc();
4306 MachineBasicBlock::iterator MII = Inst;
4308 const MCInstrDesc &InstDesc = get(Opcode);
4309 const TargetRegisterClass *Src0RC = Src0.isReg() ?
4310 MRI.getRegClass(Src0.getReg()) :
4311 &AMDGPU::SGPR_32RegClass;
4313 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4314 const TargetRegisterClass *Src1RC = Src1.isReg() ?
4315 MRI.getRegClass(Src1.getReg()) :
4316 &AMDGPU::SGPR_32RegClass;
4318 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4320 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4321 AMDGPU::sub0, Src0SubRC);
4322 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4323 AMDGPU::sub0, Src1SubRC);
4325 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4326 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4327 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4329 unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4330 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
4334 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4335 AMDGPU::sub1, Src0SubRC);
4336 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4337 AMDGPU::sub1, Src1SubRC);
4339 unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4340 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
4344 unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4345 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4347 .addImm(AMDGPU::sub0)
4349 .addImm(AMDGPU::sub1);
4351 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4353 // Try to legalize the operands in case we need to swap the order to keep it
4355 legalizeOperands(LoHalf);
4356 legalizeOperands(HiHalf);
4358 // Move all users of this moved vlaue.
4359 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4362 void SIInstrInfo::splitScalar64BitBCNT(
4363 SetVectorType &Worklist, MachineInstr &Inst) const {
4364 MachineBasicBlock &MBB = *Inst.getParent();
4365 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4367 MachineBasicBlock::iterator MII = Inst;
4368 DebugLoc DL = Inst.getDebugLoc();
4370 MachineOperand &Dest = Inst.getOperand(0);
4371 MachineOperand &Src = Inst.getOperand(1);
4373 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
4374 const TargetRegisterClass *SrcRC = Src.isReg() ?
4375 MRI.getRegClass(Src.getReg()) :
4376 &AMDGPU::SGPR_32RegClass;
4378 unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4379 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4381 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
4383 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4384 AMDGPU::sub0, SrcSubRC);
4385 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4386 AMDGPU::sub1, SrcSubRC);
4388 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
4390 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
4392 MRI.replaceRegWith(Dest.getReg(), ResultReg);
4394 // We don't need to legalize operands here. src0 for etiher instruction can be
4395 // an SGPR, and the second input is unused or determined here.
4396 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4399 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
4400 MachineInstr &Inst) const {
4401 MachineBasicBlock &MBB = *Inst.getParent();
4402 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4403 MachineBasicBlock::iterator MII = Inst;
4404 DebugLoc DL = Inst.getDebugLoc();
4406 MachineOperand &Dest = Inst.getOperand(0);
4407 uint32_t Imm = Inst.getOperand(2).getImm();
4408 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4409 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4413 // Only sext_inreg cases handled.
4414 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
4415 Offset == 0 && "Not implemented");
4417 if (BitWidth < 32) {
4418 unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4419 unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4420 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4422 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
4423 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
4427 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
4431 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4433 .addImm(AMDGPU::sub0)
4435 .addImm(AMDGPU::sub1);
4437 MRI.replaceRegWith(Dest.getReg(), ResultReg);
4438 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4442 MachineOperand &Src = Inst.getOperand(1);
4443 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4444 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4446 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
4448 .addReg(Src.getReg(), 0, AMDGPU::sub0);
4450 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4451 .addReg(Src.getReg(), 0, AMDGPU::sub0)
4452 .addImm(AMDGPU::sub0)
4454 .addImm(AMDGPU::sub1);
4456 MRI.replaceRegWith(Dest.getReg(), ResultReg);
4457 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4460 void SIInstrInfo::addUsersToMoveToVALUWorklist(
4462 MachineRegisterInfo &MRI,
4463 SetVectorType &Worklist) const {
4464 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
4465 E = MRI.use_end(); I != E;) {
4466 MachineInstr &UseMI = *I->getParent();
4467 if (!canReadVGPR(UseMI, I.getOperandNo())) {
4468 Worklist.insert(&UseMI);
4472 } while (I != E && I->getParent() == &UseMI);
4479 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
4480 MachineRegisterInfo &MRI,
4481 MachineInstr &Inst) const {
4482 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4483 MachineBasicBlock *MBB = Inst.getParent();
4484 MachineOperand &Src0 = Inst.getOperand(1);
4485 MachineOperand &Src1 = Inst.getOperand(2);
4486 const DebugLoc &DL = Inst.getDebugLoc();
4488 switch (Inst.getOpcode()) {
4489 case AMDGPU::S_PACK_LL_B32_B16: {
4490 unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4491 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4493 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
4495 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4498 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
4499 .addReg(ImmReg, RegState::Kill)
4502 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg)
4505 .addReg(TmpReg, RegState::Kill);
4508 case AMDGPU::S_PACK_LH_B32_B16: {
4509 unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4510 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4512 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg)
4513 .addReg(ImmReg, RegState::Kill)
4518 case AMDGPU::S_PACK_HH_B32_B16: {
4519 unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4520 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4521 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
4524 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4525 .addImm(0xffff0000);
4526 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg)
4528 .addReg(ImmReg, RegState::Kill)
4529 .addReg(TmpReg, RegState::Kill);
4533 llvm_unreachable("unhandled s_pack_* instruction");
4536 MachineOperand &Dest = Inst.getOperand(0);
4537 MRI.replaceRegWith(Dest.getReg(), ResultReg);
4538 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4541 void SIInstrInfo::addSCCDefUsersToVALUWorklist(
4542 MachineInstr &SCCDefInst, SetVectorType &Worklist) const {
4543 // This assumes that all the users of SCC are in the same block
4545 for (MachineInstr &MI :
4546 make_range(MachineBasicBlock::iterator(SCCDefInst),
4547 SCCDefInst.getParent()->end())) {
4548 // Exit if we find another SCC def.
4549 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC) != -1)
4552 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC) != -1)
4553 Worklist.insert(&MI);
4557 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
4558 const MachineInstr &Inst) const {
4559 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
4561 switch (Inst.getOpcode()) {
4562 // For target instructions, getOpRegClass just returns the virtual register
4563 // class associated with the operand, so we need to find an equivalent VGPR
4564 // register class in order to move the instruction to the VALU.
4567 case AMDGPU::REG_SEQUENCE:
4568 case AMDGPU::INSERT_SUBREG:
4571 if (RI.hasVGPRs(NewDstRC))
4574 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
4583 // Find the one SGPR operand we are allowed to use.
4584 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
4585 int OpIndices[3]) const {
4586 const MCInstrDesc &Desc = MI.getDesc();
4588 // Find the one SGPR operand we are allowed to use.
4590 // First we need to consider the instruction's operand requirements before
4591 // legalizing. Some operands are required to be SGPRs, such as implicit uses
4592 // of VCC, but we are still bound by the constant bus requirement to only use
4595 // If the operand's class is an SGPR, we can never move it.
4597 unsigned SGPRReg = findImplicitSGPRRead(MI);
4598 if (SGPRReg != AMDGPU::NoRegister)
4601 unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
4602 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4604 for (unsigned i = 0; i < 3; ++i) {
4605 int Idx = OpIndices[i];
4609 const MachineOperand &MO = MI.getOperand(Idx);
4613 // Is this operand statically required to be an SGPR based on the operand
4615 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
4616 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
4620 // If this could be a VGPR or an SGPR, Check the dynamic register class.
4621 unsigned Reg = MO.getReg();
4622 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
4623 if (RI.isSGPRClass(RegRC))
4627 // We don't have a required SGPR operand, so we have a bit more freedom in
4628 // selecting operands to move.
4630 // Try to select the most used SGPR. If an SGPR is equal to one of the
4631 // others, we choose that.
4634 // V_FMA_F32 v0, s0, s0, s0 -> No moves
4635 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
4637 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
4640 if (UsedSGPRs[0] != AMDGPU::NoRegister) {
4641 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
4642 SGPRReg = UsedSGPRs[0];
4645 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
4646 if (UsedSGPRs[1] == UsedSGPRs[2])
4647 SGPRReg = UsedSGPRs[1];
4653 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
4654 unsigned OperandName) const {
4655 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
4659 return &MI.getOperand(Idx);
4662 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
4663 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
4664 if (ST.isAmdHsaOS()) {
4665 // Set ATC = 1. GFX9 doesn't have this bit.
4666 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
4667 RsrcDataFormat |= (1ULL << 56);
4669 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
4670 // BTW, it disables TC L2 and therefore decreases performance.
4671 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
4672 RsrcDataFormat |= (2ULL << 59);
4675 return RsrcDataFormat;
4678 uint64_t SIInstrInfo::getScratchRsrcWords23() const {
4679 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
4680 AMDGPU::RSRC_TID_ENABLE |
4681 0xffffffff; // Size;
4683 // GFX9 doesn't have ELEMENT_SIZE.
4684 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4685 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
4686 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
4689 // IndexStride = 64.
4690 Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
4692 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
4693 // Clear them unless we want a huge stride.
4694 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
4695 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
4700 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
4701 unsigned Opc = MI.getOpcode();
4706 bool SIInstrInfo::isHighLatencyInstruction(const MachineInstr &MI) const {
4707 unsigned Opc = MI.getOpcode();
4709 return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
4712 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
4713 int &FrameIndex) const {
4714 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4715 if (!Addr || !Addr->isFI())
4716 return AMDGPU::NoRegister;
4718 assert(!MI.memoperands_empty() &&
4719 (*MI.memoperands_begin())->getAddrSpace() == ST.getAMDGPUAS().PRIVATE_ADDRESS);
4721 FrameIndex = Addr->getIndex();
4722 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
4725 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
4726 int &FrameIndex) const {
4727 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
4728 assert(Addr && Addr->isFI());
4729 FrameIndex = Addr->getIndex();
4730 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
4733 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
4734 int &FrameIndex) const {
4736 return AMDGPU::NoRegister;
4738 if (isMUBUF(MI) || isVGPRSpill(MI))
4739 return isStackAccess(MI, FrameIndex);
4741 if (isSGPRSpill(MI))
4742 return isSGPRStackAccess(MI, FrameIndex);
4744 return AMDGPU::NoRegister;
4747 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
4748 int &FrameIndex) const {
4750 return AMDGPU::NoRegister;
4752 if (isMUBUF(MI) || isVGPRSpill(MI))
4753 return isStackAccess(MI, FrameIndex);
4755 if (isSGPRSpill(MI))
4756 return isSGPRStackAccess(MI, FrameIndex);
4758 return AMDGPU::NoRegister;
4761 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const {
4763 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4764 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4765 while (++I != E && I->isInsideBundle()) {
4766 assert(!I->isBundle() && "No nested bundle!");
4767 Size += getInstSizeInBytes(*I);
4773 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
4774 unsigned Opc = MI.getOpcode();
4775 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
4776 unsigned DescSize = Desc.getSize();
4778 // If we have a definitive size, we can use it. Otherwise we need to inspect
4779 // the operands to know the size.
4781 // FIXME: Instructions that have a base 32-bit encoding report their size as
4782 // 4, even though they are really 8 bytes if they have a literal operand.
4783 if (DescSize != 0 && DescSize != 4)
4786 if (isFixedSize(MI))
4789 // 4-byte instructions may have a 32-bit literal encoded after them. Check
4790 // operands that coud ever be literals.
4791 if (isVALU(MI) || isSALU(MI)) {
4792 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4794 return 4; // No operands.
4796 if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx]))
4799 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
4803 if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx]))
4813 case TargetOpcode::IMPLICIT_DEF:
4814 case TargetOpcode::KILL:
4815 case TargetOpcode::DBG_VALUE:
4816 case TargetOpcode::EH_LABEL:
4818 case TargetOpcode::BUNDLE:
4819 return getInstBundleSize(MI);
4820 case TargetOpcode::INLINEASM: {
4821 const MachineFunction *MF = MI.getParent()->getParent();
4822 const char *AsmStr = MI.getOperand(0).getSymbolName();
4823 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
4826 llvm_unreachable("unable to find instruction size");
4830 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
4834 if (MI.memoperands_empty())
4837 for (const MachineMemOperand *MMO : MI.memoperands()) {
4838 if (MMO->getAddrSpace() == ST.getAMDGPUAS().FLAT_ADDRESS)
4844 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const {
4845 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
4848 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
4849 MachineBasicBlock *IfEnd) const {
4850 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator();
4851 assert(TI != IfEntry->end());
4853 MachineInstr *Branch = &(*TI);
4854 MachineFunction *MF = IfEntry->getParent();
4855 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
4857 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
4858 unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4859 MachineInstr *SIIF =
4860 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
4861 .add(Branch->getOperand(0))
4862 .add(Branch->getOperand(1));
4863 MachineInstr *SIEND =
4864 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
4868 IfEntry->insert(IfEntry->end(), SIIF);
4869 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND);
4873 void SIInstrInfo::convertNonUniformLoopRegion(
4874 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
4875 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator();
4876 // We expect 2 terminators, one conditional and one unconditional.
4877 assert(TI != LoopEnd->end());
4879 MachineInstr *Branch = &(*TI);
4880 MachineFunction *MF = LoopEnd->getParent();
4881 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
4883 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
4885 unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4886 unsigned BackEdgeReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4887 MachineInstrBuilder HeaderPHIBuilder =
4888 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg);
4889 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(),
4890 E = LoopEntry->pred_end();
4892 if (*PI == LoopEnd) {
4893 HeaderPHIBuilder.addReg(BackEdgeReg);
4895 MachineBasicBlock *PMBB = *PI;
4896 unsigned ZeroReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4897 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(),
4899 HeaderPHIBuilder.addReg(ZeroReg);
4901 HeaderPHIBuilder.addMBB(*PI);
4903 MachineInstr *HeaderPhi = HeaderPHIBuilder;
4904 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(),
4905 get(AMDGPU::SI_IF_BREAK), BackEdgeReg)
4907 .add(Branch->getOperand(0));
4908 MachineInstr *SILOOP =
4909 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP))
4910 .addReg(BackEdgeReg)
4913 LoopEntry->insert(LoopEntry->begin(), HeaderPhi);
4915 LoopEnd->insert(LoopEnd->end(), SIIFBREAK);
4916 LoopEnd->insert(LoopEnd->end(), SILOOP);
4920 ArrayRef<std::pair<int, const char *>>
4921 SIInstrInfo::getSerializableTargetIndices() const {
4922 static const std::pair<int, const char *> TargetIndices[] = {
4923 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
4924 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
4925 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
4926 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
4927 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
4928 return makeArrayRef(TargetIndices);
4931 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
4932 /// post-RA version of misched uses CreateTargetMIHazardRecognizer.
4933 ScheduleHazardRecognizer *
4934 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
4935 const ScheduleDAG *DAG) const {
4936 return new GCNHazardRecognizer(DAG->MF);
4939 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
4941 ScheduleHazardRecognizer *
4942 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
4943 return new GCNHazardRecognizer(MF);
4946 std::pair<unsigned, unsigned>
4947 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
4948 return std::make_pair(TF & MO_MASK, TF & ~MO_MASK);
4951 ArrayRef<std::pair<unsigned, const char *>>
4952 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
4953 static const std::pair<unsigned, const char *> TargetFlags[] = {
4954 { MO_GOTPCREL, "amdgpu-gotprel" },
4955 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" },
4956 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" },
4957 { MO_REL32_LO, "amdgpu-rel32-lo" },
4958 { MO_REL32_HI, "amdgpu-rel32-hi" }
4961 return makeArrayRef(TargetFlags);
4964 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const {
4965 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
4966 MI.modifiesRegister(AMDGPU::EXEC, &RI);
4970 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
4971 MachineBasicBlock::iterator I,
4973 unsigned DestReg) const {
4974 if (ST.hasAddNoCarry())
4975 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
4977 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4978 unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4979 MRI.setRegAllocationHint(UnusedCarry, 0, AMDGPU::VCC);
4981 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
4982 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
4985 bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
4987 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
4988 case AMDGPU::SI_KILL_I1_TERMINATOR:
4995 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
4997 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
4998 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
4999 case AMDGPU::SI_KILL_I1_PSEUDO:
5000 return get(AMDGPU::SI_KILL_I1_TERMINATOR);
5002 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
5006 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
5010 // Check that it is using a buffer resource.
5011 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
5012 if (Idx == -1) // e.g. s_memtime
5015 const auto RCID = MI.getDesc().OpInfo[Idx].RegClass;
5016 return RCID == AMDGPU::SReg_128RegClassID;
5019 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td
5020 enum SIEncodingFamily {
5029 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) {
5030 switch (ST.getGeneration()) {
5033 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
5034 case AMDGPUSubtarget::SEA_ISLANDS:
5035 return SIEncodingFamily::SI;
5036 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
5037 case AMDGPUSubtarget::GFX9:
5038 return SIEncodingFamily::VI;
5040 llvm_unreachable("Unknown subtarget generation!");
5043 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
5044 SIEncodingFamily Gen = subtargetEncodingFamily(ST);
5046 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
5047 ST.getGeneration() >= AMDGPUSubtarget::GFX9)
5048 Gen = SIEncodingFamily::GFX9;
5050 if (get(Opcode).TSFlags & SIInstrFlags::SDWA)
5051 Gen = ST.getGeneration() == AMDGPUSubtarget::GFX9 ? SIEncodingFamily::SDWA9
5052 : SIEncodingFamily::SDWA;
5053 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
5054 // subtarget has UnpackedD16VMem feature.
5055 // TODO: remove this when we discard GFX80 encoding.
5056 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))
5057 Gen = SIEncodingFamily::GFX80;
5059 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
5061 // -1 means that Opcode is already a native instruction.
5065 // (uint16_t)-1 means that Opcode is a pseudo instruction that has
5066 // no encoding in the given subtarget generation.
5067 if (MCOp == (uint16_t)-1)