1 //===-- SIInstrInfo.cpp - SI Instruction Information ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief SI Implementation of TargetInstrInfo.
13 //===----------------------------------------------------------------------===//
15 #include "SIInstrInfo.h"
16 #include "AMDGPUTargetMachine.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "llvm/CodeGen/MachineFrameInfo.h"
21 #include "llvm/CodeGen/MachineInstrBuilder.h"
22 #include "llvm/CodeGen/MachineRegisterInfo.h"
23 #include "llvm/CodeGen/ScheduleDAG.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/CodeGen/RegisterScavenging.h"
26 #include "llvm/MC/MCInstrDesc.h"
27 #include "llvm/Support/Debug.h"
31 SIInstrInfo::SIInstrInfo(const SISubtarget &ST)
32 : AMDGPUInstrInfo(ST), RI(), ST(ST) {}
34 //===----------------------------------------------------------------------===//
35 // TargetInstrInfo callbacks
36 //===----------------------------------------------------------------------===//
38 static unsigned getNumOperandsNoGlue(SDNode *Node) {
39 unsigned N = Node->getNumOperands();
40 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
45 static SDValue findChainOperand(SDNode *Load) {
46 SDValue LastOp = Load->getOperand(getNumOperandsNoGlue(Load) - 1);
47 assert(LastOp.getValueType() == MVT::Other && "Chain missing from load node");
51 /// \brief Returns true if both nodes have the same value for the given
52 /// operand \p Op, or if both nodes do not have this operand.
53 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
54 unsigned Opc0 = N0->getMachineOpcode();
55 unsigned Opc1 = N1->getMachineOpcode();
57 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
58 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
60 if (Op0Idx == -1 && Op1Idx == -1)
64 if ((Op0Idx == -1 && Op1Idx != -1) ||
65 (Op1Idx == -1 && Op0Idx != -1))
68 // getNamedOperandIdx returns the index for the MachineInstr's operands,
69 // which includes the result as the first operand. We are indexing into the
70 // MachineSDNode's operands, so we need to skip the result operand to get
75 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
78 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
79 AliasAnalysis *AA) const {
80 // TODO: The generic check fails for VALU instructions that should be
81 // rematerializable due to implicit reads of exec. We really want all of the
82 // generic logic for this except for this.
83 switch (MI.getOpcode()) {
84 case AMDGPU::V_MOV_B32_e32:
85 case AMDGPU::V_MOV_B32_e64:
86 case AMDGPU::V_MOV_B64_PSEUDO:
93 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
95 int64_t &Offset1) const {
96 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
99 unsigned Opc0 = Load0->getMachineOpcode();
100 unsigned Opc1 = Load1->getMachineOpcode();
102 // Make sure both are actually loads.
103 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
106 if (isDS(Opc0) && isDS(Opc1)) {
108 // FIXME: Handle this case:
109 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
113 if (Load0->getOperand(1) != Load1->getOperand(1))
117 if (findChainOperand(Load0) != findChainOperand(Load1))
120 // Skip read2 / write2 variants for simplicity.
121 // TODO: We should report true if the used offsets are adjacent (excluded
123 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 ||
124 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1)
127 Offset0 = cast<ConstantSDNode>(Load0->getOperand(2))->getZExtValue();
128 Offset1 = cast<ConstantSDNode>(Load1->getOperand(2))->getZExtValue();
132 if (isSMRD(Opc0) && isSMRD(Opc1)) {
133 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1));
136 if (Load0->getOperand(0) != Load1->getOperand(0))
139 const ConstantSDNode *Load0Offset =
140 dyn_cast<ConstantSDNode>(Load0->getOperand(1));
141 const ConstantSDNode *Load1Offset =
142 dyn_cast<ConstantSDNode>(Load1->getOperand(1));
144 if (!Load0Offset || !Load1Offset)
148 if (findChainOperand(Load0) != findChainOperand(Load1))
151 Offset0 = Load0Offset->getZExtValue();
152 Offset1 = Load1Offset->getZExtValue();
156 // MUBUF and MTBUF can access the same addresses.
157 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
159 // MUBUF and MTBUF have vaddr at different indices.
160 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
161 findChainOperand(Load0) != findChainOperand(Load1) ||
162 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
163 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
166 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
167 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
169 if (OffIdx0 == -1 || OffIdx1 == -1)
172 // getNamedOperandIdx returns the index for MachineInstrs. Since they
173 // inlcude the output in the operand list, but SDNodes don't, we need to
174 // subtract the index by one.
178 SDValue Off0 = Load0->getOperand(OffIdx0);
179 SDValue Off1 = Load1->getOperand(OffIdx1);
181 // The offset might be a FrameIndexSDNode.
182 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
185 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
186 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
193 static bool isStride64(unsigned Opc) {
195 case AMDGPU::DS_READ2ST64_B32:
196 case AMDGPU::DS_READ2ST64_B64:
197 case AMDGPU::DS_WRITE2ST64_B32:
198 case AMDGPU::DS_WRITE2ST64_B64:
205 bool SIInstrInfo::getMemOpBaseRegImmOfs(MachineInstr &LdSt, unsigned &BaseReg,
207 const TargetRegisterInfo *TRI) const {
208 unsigned Opc = LdSt.getOpcode();
211 const MachineOperand *OffsetImm =
212 getNamedOperand(LdSt, AMDGPU::OpName::offset);
214 // Normal, single offset LDS instruction.
215 const MachineOperand *AddrReg =
216 getNamedOperand(LdSt, AMDGPU::OpName::addr);
218 BaseReg = AddrReg->getReg();
219 Offset = OffsetImm->getImm();
223 // The 2 offset instructions use offset0 and offset1 instead. We can treat
224 // these as a load with a single offset if the 2 offsets are consecutive. We
225 // will use this for some partially aligned loads.
226 const MachineOperand *Offset0Imm =
227 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
228 const MachineOperand *Offset1Imm =
229 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
231 uint8_t Offset0 = Offset0Imm->getImm();
232 uint8_t Offset1 = Offset1Imm->getImm();
234 if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
235 // Each of these offsets is in element sized units, so we need to convert
236 // to bytes of the individual reads.
240 EltSize = getOpRegClass(LdSt, 0)->getSize() / 2;
242 assert(LdSt.mayStore());
243 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
244 EltSize = getOpRegClass(LdSt, Data0Idx)->getSize();
250 const MachineOperand *AddrReg =
251 getNamedOperand(LdSt, AMDGPU::OpName::addr);
252 BaseReg = AddrReg->getReg();
253 Offset = EltSize * Offset0;
260 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
261 if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::soffset) != -1)
264 const MachineOperand *AddrReg =
265 getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
269 const MachineOperand *OffsetImm =
270 getNamedOperand(LdSt, AMDGPU::OpName::offset);
271 BaseReg = AddrReg->getReg();
272 Offset = OffsetImm->getImm();
277 const MachineOperand *OffsetImm =
278 getNamedOperand(LdSt, AMDGPU::OpName::offset);
282 const MachineOperand *SBaseReg =
283 getNamedOperand(LdSt, AMDGPU::OpName::sbase);
284 BaseReg = SBaseReg->getReg();
285 Offset = OffsetImm->getImm();
290 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::addr);
291 BaseReg = AddrReg->getReg();
299 bool SIInstrInfo::shouldClusterMemOps(MachineInstr &FirstLdSt,
300 MachineInstr &SecondLdSt,
301 unsigned NumLoads) const {
302 const MachineOperand *FirstDst = nullptr;
303 const MachineOperand *SecondDst = nullptr;
305 if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
306 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
307 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
310 if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
311 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
312 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
315 if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
316 (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt))) {
317 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
318 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
321 if (!FirstDst || !SecondDst)
324 // Try to limit clustering based on the total number of bytes loaded
325 // rather than the number of instructions. This is done to help reduce
326 // register pressure. The method used is somewhat inexact, though,
327 // because it assumes that all loads in the cluster will load the
328 // same number of bytes as FirstLdSt.
330 // The unit of this value is bytes.
331 // FIXME: This needs finer tuning.
332 unsigned LoadClusterThreshold = 16;
334 const MachineRegisterInfo &MRI =
335 FirstLdSt.getParent()->getParent()->getRegInfo();
336 const TargetRegisterClass *DstRC = MRI.getRegClass(FirstDst->getReg());
338 return (NumLoads * DstRC->getSize()) <= LoadClusterThreshold;
341 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
342 MachineBasicBlock::iterator MI,
343 const DebugLoc &DL, unsigned DestReg,
344 unsigned SrcReg, bool KillSrc) const {
346 // If we are trying to copy to or from SCC, there is a bug somewhere else in
347 // the backend. While it may be theoretically possible to do this, it should
348 // never be necessary.
349 assert(DestReg != AMDGPU::SCC && SrcReg != AMDGPU::SCC);
351 static const int16_t Sub0_15[] = {
352 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
353 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
354 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
355 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
358 static const int16_t Sub0_15_64[] = {
359 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
360 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
361 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
362 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
365 static const int16_t Sub0_7[] = {
366 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
367 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
370 static const int16_t Sub0_7_64[] = {
371 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
372 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
375 static const int16_t Sub0_3[] = {
376 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
379 static const int16_t Sub0_3_64[] = {
380 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
383 static const int16_t Sub0_2[] = {
384 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2,
387 static const int16_t Sub0_1[] = {
388 AMDGPU::sub0, AMDGPU::sub1,
392 ArrayRef<int16_t> SubIndices;
394 if (AMDGPU::SReg_32RegClass.contains(DestReg)) {
395 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
396 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
397 .addReg(SrcReg, getKillRegState(KillSrc));
400 } else if (AMDGPU::SReg_64RegClass.contains(DestReg)) {
401 if (DestReg == AMDGPU::VCC) {
402 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
403 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
404 .addReg(SrcReg, getKillRegState(KillSrc));
406 // FIXME: Hack until VReg_1 removed.
407 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
408 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_I32_e32))
410 .addReg(SrcReg, getKillRegState(KillSrc));
416 assert(AMDGPU::SReg_64RegClass.contains(SrcReg));
417 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
418 .addReg(SrcReg, getKillRegState(KillSrc));
421 } else if (AMDGPU::SReg_128RegClass.contains(DestReg)) {
422 assert(AMDGPU::SReg_128RegClass.contains(SrcReg));
423 Opcode = AMDGPU::S_MOV_B64;
424 SubIndices = Sub0_3_64;
426 } else if (AMDGPU::SReg_256RegClass.contains(DestReg)) {
427 assert(AMDGPU::SReg_256RegClass.contains(SrcReg));
428 Opcode = AMDGPU::S_MOV_B64;
429 SubIndices = Sub0_7_64;
431 } else if (AMDGPU::SReg_512RegClass.contains(DestReg)) {
432 assert(AMDGPU::SReg_512RegClass.contains(SrcReg));
433 Opcode = AMDGPU::S_MOV_B64;
434 SubIndices = Sub0_15_64;
436 } else if (AMDGPU::VGPR_32RegClass.contains(DestReg)) {
437 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
438 AMDGPU::SReg_32RegClass.contains(SrcReg));
439 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
440 .addReg(SrcReg, getKillRegState(KillSrc));
443 } else if (AMDGPU::VReg_64RegClass.contains(DestReg)) {
444 assert(AMDGPU::VReg_64RegClass.contains(SrcReg) ||
445 AMDGPU::SReg_64RegClass.contains(SrcReg));
446 Opcode = AMDGPU::V_MOV_B32_e32;
449 } else if (AMDGPU::VReg_96RegClass.contains(DestReg)) {
450 assert(AMDGPU::VReg_96RegClass.contains(SrcReg));
451 Opcode = AMDGPU::V_MOV_B32_e32;
454 } else if (AMDGPU::VReg_128RegClass.contains(DestReg)) {
455 assert(AMDGPU::VReg_128RegClass.contains(SrcReg) ||
456 AMDGPU::SReg_128RegClass.contains(SrcReg));
457 Opcode = AMDGPU::V_MOV_B32_e32;
460 } else if (AMDGPU::VReg_256RegClass.contains(DestReg)) {
461 assert(AMDGPU::VReg_256RegClass.contains(SrcReg) ||
462 AMDGPU::SReg_256RegClass.contains(SrcReg));
463 Opcode = AMDGPU::V_MOV_B32_e32;
466 } else if (AMDGPU::VReg_512RegClass.contains(DestReg)) {
467 assert(AMDGPU::VReg_512RegClass.contains(SrcReg) ||
468 AMDGPU::SReg_512RegClass.contains(SrcReg));
469 Opcode = AMDGPU::V_MOV_B32_e32;
470 SubIndices = Sub0_15;
473 llvm_unreachable("Can't copy register!");
476 bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
478 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
481 SubIdx = SubIndices[Idx];
483 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
485 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
486 get(Opcode), RI.getSubReg(DestReg, SubIdx));
488 Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
490 if (Idx == SubIndices.size() - 1)
491 Builder.addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);
494 Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
496 Builder.addReg(SrcReg, RegState::Implicit);
500 int SIInstrInfo::commuteOpcode(const MachineInstr &MI) const {
501 const unsigned Opcode = MI.getOpcode();
505 // Try to map original to commuted opcode
506 NewOpc = AMDGPU::getCommuteRev(Opcode);
508 // Check if the commuted (REV) opcode exists on the target.
509 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
511 // Try to map commuted to original opcode
512 NewOpc = AMDGPU::getCommuteOrig(Opcode);
514 // Check if the original (non-REV) opcode exists on the target.
515 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
520 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
522 if (DstRC->getSize() == 4) {
523 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
524 } else if (DstRC->getSize() == 8 && RI.isSGPRClass(DstRC)) {
525 return AMDGPU::S_MOV_B64;
526 } else if (DstRC->getSize() == 8 && !RI.isSGPRClass(DstRC)) {
527 return AMDGPU::V_MOV_B64_PSEUDO;
532 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
535 return AMDGPU::SI_SPILL_S32_SAVE;
537 return AMDGPU::SI_SPILL_S64_SAVE;
539 return AMDGPU::SI_SPILL_S128_SAVE;
541 return AMDGPU::SI_SPILL_S256_SAVE;
543 return AMDGPU::SI_SPILL_S512_SAVE;
545 llvm_unreachable("unknown register size");
549 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
552 return AMDGPU::SI_SPILL_V32_SAVE;
554 return AMDGPU::SI_SPILL_V64_SAVE;
556 return AMDGPU::SI_SPILL_V96_SAVE;
558 return AMDGPU::SI_SPILL_V128_SAVE;
560 return AMDGPU::SI_SPILL_V256_SAVE;
562 return AMDGPU::SI_SPILL_V512_SAVE;
564 llvm_unreachable("unknown register size");
568 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
569 MachineBasicBlock::iterator MI,
570 unsigned SrcReg, bool isKill,
572 const TargetRegisterClass *RC,
573 const TargetRegisterInfo *TRI) const {
574 MachineFunction *MF = MBB.getParent();
575 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
576 MachineFrameInfo *FrameInfo = MF->getFrameInfo();
577 DebugLoc DL = MBB.findDebugLoc(MI);
579 unsigned Size = FrameInfo->getObjectSize(FrameIndex);
580 unsigned Align = FrameInfo->getObjectAlignment(FrameIndex);
581 MachinePointerInfo PtrInfo
582 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
583 MachineMemOperand *MMO
584 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
587 if (RI.isSGPRClass(RC)) {
588 MFI->setHasSpilledSGPRs();
590 if (TargetRegisterInfo::isVirtualRegister(SrcReg) && RC->getSize() == 4) {
591 // m0 may not be allowed for readlane.
592 MachineRegisterInfo &MRI = MF->getRegInfo();
593 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
596 // We are only allowed to create one new instruction when spilling
597 // registers, so we need to use pseudo instruction for spilling
599 unsigned Opcode = getSGPRSpillSaveOpcode(RC->getSize());
600 BuildMI(MBB, MI, DL, get(Opcode))
601 .addReg(SrcReg, getKillRegState(isKill)) // src
602 .addFrameIndex(FrameIndex) // frame_idx
608 if (!ST.isVGPRSpillingEnabled(*MF->getFunction())) {
609 LLVMContext &Ctx = MF->getFunction()->getContext();
610 Ctx.emitError("SIInstrInfo::storeRegToStackSlot - Do not know how to"
612 BuildMI(MBB, MI, DL, get(AMDGPU::KILL))
618 assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
620 unsigned Opcode = getVGPRSpillSaveOpcode(RC->getSize());
621 MFI->setHasSpilledVGPRs();
622 BuildMI(MBB, MI, DL, get(Opcode))
623 .addReg(SrcReg, getKillRegState(isKill)) // src
624 .addFrameIndex(FrameIndex) // frame_idx
625 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
626 .addReg(MFI->getScratchWaveOffsetReg()) // scratch_offset
631 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
634 return AMDGPU::SI_SPILL_S32_RESTORE;
636 return AMDGPU::SI_SPILL_S64_RESTORE;
638 return AMDGPU::SI_SPILL_S128_RESTORE;
640 return AMDGPU::SI_SPILL_S256_RESTORE;
642 return AMDGPU::SI_SPILL_S512_RESTORE;
644 llvm_unreachable("unknown register size");
648 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
651 return AMDGPU::SI_SPILL_V32_RESTORE;
653 return AMDGPU::SI_SPILL_V64_RESTORE;
655 return AMDGPU::SI_SPILL_V96_RESTORE;
657 return AMDGPU::SI_SPILL_V128_RESTORE;
659 return AMDGPU::SI_SPILL_V256_RESTORE;
661 return AMDGPU::SI_SPILL_V512_RESTORE;
663 llvm_unreachable("unknown register size");
667 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
668 MachineBasicBlock::iterator MI,
669 unsigned DestReg, int FrameIndex,
670 const TargetRegisterClass *RC,
671 const TargetRegisterInfo *TRI) const {
672 MachineFunction *MF = MBB.getParent();
673 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
674 MachineFrameInfo *FrameInfo = MF->getFrameInfo();
675 DebugLoc DL = MBB.findDebugLoc(MI);
676 unsigned Align = FrameInfo->getObjectAlignment(FrameIndex);
677 unsigned Size = FrameInfo->getObjectSize(FrameIndex);
679 MachinePointerInfo PtrInfo
680 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
682 MachineMemOperand *MMO = MF->getMachineMemOperand(
683 PtrInfo, MachineMemOperand::MOLoad, Size, Align);
685 if (RI.isSGPRClass(RC)) {
686 // FIXME: Maybe this should not include a memoperand because it will be
687 // lowered to non-memory instructions.
688 unsigned Opcode = getSGPRSpillRestoreOpcode(RC->getSize());
690 if (TargetRegisterInfo::isVirtualRegister(DestReg) && RC->getSize() == 4) {
691 // m0 may not be allowed for readlane.
692 MachineRegisterInfo &MRI = MF->getRegInfo();
693 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
696 BuildMI(MBB, MI, DL, get(Opcode), DestReg)
697 .addFrameIndex(FrameIndex) // frame_idx
703 if (!ST.isVGPRSpillingEnabled(*MF->getFunction())) {
704 LLVMContext &Ctx = MF->getFunction()->getContext();
705 Ctx.emitError("SIInstrInfo::loadRegFromStackSlot - Do not know how to"
706 " restore register");
707 BuildMI(MBB, MI, DL, get(AMDGPU::IMPLICIT_DEF), DestReg);
712 assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
714 unsigned Opcode = getVGPRSpillRestoreOpcode(RC->getSize());
715 BuildMI(MBB, MI, DL, get(Opcode), DestReg)
716 .addFrameIndex(FrameIndex) // frame_idx
717 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
718 .addReg(MFI->getScratchWaveOffsetReg()) // scratch_offset
723 /// \param @Offset Offset in bytes of the FrameIndex being spilled
724 unsigned SIInstrInfo::calculateLDSSpillAddress(
725 MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
726 unsigned FrameOffset, unsigned Size) const {
727 MachineFunction *MF = MBB.getParent();
728 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
729 const SISubtarget &ST = MF->getSubtarget<SISubtarget>();
730 const SIRegisterInfo *TRI = ST.getRegisterInfo();
731 DebugLoc DL = MBB.findDebugLoc(MI);
732 unsigned WorkGroupSize = MFI->getMaximumWorkGroupSize(*MF);
733 unsigned WavefrontSize = ST.getWavefrontSize();
735 unsigned TIDReg = MFI->getTIDReg();
736 if (!MFI->hasCalculatedTID()) {
737 MachineBasicBlock &Entry = MBB.getParent()->front();
738 MachineBasicBlock::iterator Insert = Entry.front();
739 DebugLoc DL = Insert->getDebugLoc();
741 TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
743 if (TIDReg == AMDGPU::NoRegister)
746 if (!AMDGPU::isShader(MF->getFunction()->getCallingConv()) &&
747 WorkGroupSize > WavefrontSize) {
750 = TRI->getPreloadedValue(*MF, SIRegisterInfo::WORKGROUP_ID_X);
752 = TRI->getPreloadedValue(*MF, SIRegisterInfo::WORKGROUP_ID_Y);
754 = TRI->getPreloadedValue(*MF, SIRegisterInfo::WORKGROUP_ID_Z);
755 unsigned InputPtrReg =
756 TRI->getPreloadedValue(*MF, SIRegisterInfo::KERNARG_SEGMENT_PTR);
757 for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
758 if (!Entry.isLiveIn(Reg))
759 Entry.addLiveIn(Reg);
762 RS->enterBasicBlock(Entry);
763 // FIXME: Can we scavenge an SReg_64 and access the subregs?
764 unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
765 unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
766 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
768 .addImm(SI::KernelInputOffsets::NGROUPS_Z);
769 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
771 .addImm(SI::KernelInputOffsets::NGROUPS_Y);
773 // NGROUPS.X * NGROUPS.Y
774 BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
777 // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
778 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
781 // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
782 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
786 // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
787 BuildMI(Entry, Insert, DL, get(AMDGPU::V_ADD_I32_e32), TIDReg)
792 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
797 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
803 BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
807 MFI->setTIDReg(TIDReg);
810 // Add FrameIndex to LDS offset
811 unsigned LDSOffset = MFI->LDSSize + (FrameOffset * WorkGroupSize);
812 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), TmpReg)
819 void SIInstrInfo::insertWaitStates(MachineBasicBlock &MBB,
820 MachineBasicBlock::iterator MI,
822 DebugLoc DL = MBB.findDebugLoc(MI);
830 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
835 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
836 MachineBasicBlock::iterator MI) const {
837 insertWaitStates(MBB, MI, 1);
840 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) const {
841 switch (MI.getOpcode()) {
842 default: return 1; // FIXME: Do wait states equal cycles?
845 return MI.getOperand(0).getImm() + 1;
849 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
850 MachineBasicBlock &MBB = *MI.getParent();
851 DebugLoc DL = MBB.findDebugLoc(MI);
852 switch (MI.getOpcode()) {
853 default: return AMDGPUInstrInfo::expandPostRAPseudo(MI);
855 case AMDGPU::V_MOV_B64_PSEUDO: {
856 unsigned Dst = MI.getOperand(0).getReg();
857 unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
858 unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
860 const MachineOperand &SrcOp = MI.getOperand(1);
861 // FIXME: Will this work for 64-bit floating point immediates?
862 assert(!SrcOp.isFPImm());
864 APInt Imm(64, SrcOp.getImm());
865 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
866 .addImm(Imm.getLoBits(32).getZExtValue())
867 .addReg(Dst, RegState::Implicit | RegState::Define);
868 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
869 .addImm(Imm.getHiBits(32).getZExtValue())
870 .addReg(Dst, RegState::Implicit | RegState::Define);
872 assert(SrcOp.isReg());
873 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
874 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
875 .addReg(Dst, RegState::Implicit | RegState::Define);
876 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
877 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
878 .addReg(Dst, RegState::Implicit | RegState::Define);
880 MI.eraseFromParent();
884 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
885 unsigned Dst = MI.getOperand(0).getReg();
886 unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
887 unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
888 unsigned Src0 = MI.getOperand(1).getReg();
889 unsigned Src1 = MI.getOperand(2).getReg();
890 const MachineOperand &SrcCond = MI.getOperand(3);
892 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
893 .addReg(RI.getSubReg(Src0, AMDGPU::sub0))
894 .addReg(RI.getSubReg(Src1, AMDGPU::sub0))
895 .addReg(SrcCond.getReg())
896 .addReg(Dst, RegState::Implicit | RegState::Define);
897 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
898 .addReg(RI.getSubReg(Src0, AMDGPU::sub1))
899 .addReg(RI.getSubReg(Src1, AMDGPU::sub1))
900 .addReg(SrcCond.getReg(), getKillRegState(SrcCond.isKill()))
901 .addReg(Dst, RegState::Implicit | RegState::Define);
902 MI.eraseFromParent();
906 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
907 const SIRegisterInfo *TRI
908 = static_cast<const SIRegisterInfo *>(ST.getRegisterInfo());
909 MachineFunction &MF = *MBB.getParent();
910 unsigned Reg = MI.getOperand(0).getReg();
911 unsigned RegLo = TRI->getSubReg(Reg, AMDGPU::sub0);
912 unsigned RegHi = TRI->getSubReg(Reg, AMDGPU::sub1);
914 // Create a bundle so these instructions won't be re-ordered by the
915 // post-RA scheduler.
916 MIBundleBuilder Bundler(MBB, MI);
917 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
919 // Add 32-bit offset from this instruction to the start of the
921 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
923 .addOperand(MI.getOperand(1)));
924 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
928 llvm::finalizeBundle(MBB, Bundler.begin());
930 MI.eraseFromParent();
937 /// Commutes the operands in the given instruction.
938 /// The commutable operands are specified by their indices OpIdx0 and OpIdx1.
940 /// Do not call this method for a non-commutable instruction or for
941 /// non-commutable pair of operand indices OpIdx0 and OpIdx1.
942 /// Even though the instruction is commutable, the method may still
943 /// fail to commute the operands, null pointer is returned in such cases.
944 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
946 unsigned OpIdx1) const {
947 int CommutedOpcode = commuteOpcode(MI);
948 if (CommutedOpcode == -1)
952 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
953 MachineOperand &Src0 = MI.getOperand(Src0Idx);
958 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
960 if ((OpIdx0 != static_cast<unsigned>(Src0Idx) ||
961 OpIdx1 != static_cast<unsigned>(Src1Idx)) &&
962 (OpIdx0 != static_cast<unsigned>(Src1Idx) ||
963 OpIdx1 != static_cast<unsigned>(Src0Idx)))
966 MachineOperand &Src1 = MI.getOperand(Src1Idx);
968 if (isVOP2(MI) || isVOPC(MI)) {
969 const MCInstrDesc &InstrDesc = MI.getDesc();
970 // For VOP2 and VOPC instructions, any operand type is valid to use for
971 // src0. Make sure we can use the src0 as src1.
973 // We could be stricter here and only allow commuting if there is a reason
974 // to do so. i.e. if both operands are VGPRs there is no real benefit,
975 // although MachineCSE attempts to find matches by commuting.
976 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
977 if (!isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0))
981 MachineInstr *CommutedMI = &MI;
983 // Allow commuting instructions with Imm operands.
984 if (NewMI || !Src1.isImm() || (!isVOP2(MI) && !isVOP3(MI))) {
987 // Be sure to copy the source modifiers to the right place.
988 if (MachineOperand *Src0Mods =
989 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)) {
990 MachineOperand *Src1Mods =
991 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
993 int Src0ModsVal = Src0Mods->getImm();
994 if (!Src1Mods && Src0ModsVal != 0)
997 // XXX - This assert might be a lie. It might be useful to have a neg
998 // modifier with 0.0.
999 int Src1ModsVal = Src1Mods->getImm();
1000 assert((Src1ModsVal == 0) && "Not expecting modifiers with immediates");
1002 Src1Mods->setImm(Src0ModsVal);
1003 Src0Mods->setImm(Src1ModsVal);
1006 unsigned Reg = Src0.getReg();
1007 unsigned SubReg = Src0.getSubReg();
1009 Src0.ChangeToImmediate(Src1.getImm());
1011 llvm_unreachable("Should only have immediates");
1013 Src1.ChangeToRegister(Reg, false);
1014 Src1.setSubReg(SubReg);
1017 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx0, OpIdx1);
1021 CommutedMI->setDesc(get(CommutedOpcode));
1026 // This needs to be implemented because the source modifiers may be inserted
1027 // between the true commutable operands, and the base
1028 // TargetInstrInfo::commuteInstruction uses it.
1029 bool SIInstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx0,
1030 unsigned &SrcOpIdx1) const {
1031 const MCInstrDesc &MCID = MI.getDesc();
1032 if (!MCID.isCommutable())
1035 unsigned Opc = MI.getOpcode();
1036 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1040 // FIXME: Workaround TargetInstrInfo::commuteInstruction asserting on
1041 // immediate. Also, immediate src0 operand is not handled in
1042 // SIInstrInfo::commuteInstruction();
1043 if (!MI.getOperand(Src0Idx).isReg())
1046 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1050 MachineOperand &Src1 = MI.getOperand(Src1Idx);
1052 // SIInstrInfo::commuteInstruction() does support commuting the immediate
1053 // operand src1 in 2 and 3 operand instructions.
1054 if (!isVOP2(MI.getOpcode()) && !isVOP3(MI.getOpcode()))
1056 } else if (Src1.isReg()) {
1057 // If any source modifiers are set, the generic instruction commuting won't
1058 // understand how to copy the source modifiers.
1059 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
1060 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))
1065 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1068 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1070 case SIInstrInfo::SCC_TRUE:
1071 return AMDGPU::S_CBRANCH_SCC1;
1072 case SIInstrInfo::SCC_FALSE:
1073 return AMDGPU::S_CBRANCH_SCC0;
1074 case SIInstrInfo::VCCNZ:
1075 return AMDGPU::S_CBRANCH_VCCNZ;
1076 case SIInstrInfo::VCCZ:
1077 return AMDGPU::S_CBRANCH_VCCZ;
1078 case SIInstrInfo::EXECNZ:
1079 return AMDGPU::S_CBRANCH_EXECNZ;
1080 case SIInstrInfo::EXECZ:
1081 return AMDGPU::S_CBRANCH_EXECZ;
1083 llvm_unreachable("invalid branch predicate");
1087 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1089 case AMDGPU::S_CBRANCH_SCC0:
1091 case AMDGPU::S_CBRANCH_SCC1:
1093 case AMDGPU::S_CBRANCH_VCCNZ:
1095 case AMDGPU::S_CBRANCH_VCCZ:
1097 case AMDGPU::S_CBRANCH_EXECNZ:
1099 case AMDGPU::S_CBRANCH_EXECZ:
1106 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
1107 MachineBasicBlock *&FBB,
1108 SmallVectorImpl<MachineOperand> &Cond,
1109 bool AllowModify) const {
1110 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1115 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1116 // Unconditional Branch
1117 TBB = I->getOperand(0).getMBB();
1121 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1122 if (Pred == INVALID_BR)
1125 MachineBasicBlock *CondBB = I->getOperand(0).getMBB();
1126 Cond.push_back(MachineOperand::CreateImm(Pred));
1130 if (I == MBB.end()) {
1131 // Conditional branch followed by fall-through.
1136 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1138 FBB = I->getOperand(0).getMBB();
1145 unsigned SIInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
1146 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1149 while (I != MBB.end()) {
1150 MachineBasicBlock::iterator Next = std::next(I);
1151 I->eraseFromParent();
1159 unsigned SIInstrInfo::InsertBranch(MachineBasicBlock &MBB,
1160 MachineBasicBlock *TBB,
1161 MachineBasicBlock *FBB,
1162 ArrayRef<MachineOperand> Cond,
1163 const DebugLoc &DL) const {
1165 if (!FBB && Cond.empty()) {
1166 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1171 assert(TBB && Cond[0].isImm());
1174 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
1177 BuildMI(&MBB, DL, get(Opcode))
1184 BuildMI(&MBB, DL, get(Opcode))
1186 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1192 bool SIInstrInfo::ReverseBranchCondition(
1193 SmallVectorImpl<MachineOperand> &Cond) const {
1194 assert(Cond.size() == 1);
1195 Cond[0].setImm(-Cond[0].getImm());
1199 static void removeModOperands(MachineInstr &MI) {
1200 unsigned Opc = MI.getOpcode();
1201 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1202 AMDGPU::OpName::src0_modifiers);
1203 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1204 AMDGPU::OpName::src1_modifiers);
1205 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1206 AMDGPU::OpName::src2_modifiers);
1208 MI.RemoveOperand(Src2ModIdx);
1209 MI.RemoveOperand(Src1ModIdx);
1210 MI.RemoveOperand(Src0ModIdx);
1213 // TODO: Maybe this should be removed this and custom fold everything in
1215 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
1216 unsigned Reg, MachineRegisterInfo *MRI) const {
1217 if (!MRI->hasOneNonDBGUse(Reg))
1220 unsigned Opc = UseMI.getOpcode();
1221 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64) {
1222 // Don't fold if we are using source modifiers. The new VOP2 instructions
1224 if (hasModifiersSet(UseMI, AMDGPU::OpName::src0_modifiers) ||
1225 hasModifiersSet(UseMI, AMDGPU::OpName::src1_modifiers) ||
1226 hasModifiersSet(UseMI, AMDGPU::OpName::src2_modifiers)) {
1230 const MachineOperand &ImmOp = DefMI.getOperand(1);
1232 // If this is a free constant, there's no reason to do this.
1233 // TODO: We could fold this here instead of letting SIFoldOperands do it
1235 if (isInlineConstant(ImmOp, 4))
1238 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
1239 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
1240 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
1242 // Multiplied part is the constant: Use v_madmk_f32
1243 // We should only expect these to be on src0 due to canonicalizations.
1244 if (Src0->isReg() && Src0->getReg() == Reg) {
1245 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
1248 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
1251 // We need to swap operands 0 and 1 since madmk constant is at operand 1.
1253 const int64_t Imm = DefMI.getOperand(1).getImm();
1255 // FIXME: This would be a lot easier if we could return a new instruction
1256 // instead of having to modify in place.
1258 // Remove these first since they are at the end.
1259 UseMI.RemoveOperand(
1260 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
1261 UseMI.RemoveOperand(
1262 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
1264 unsigned Src1Reg = Src1->getReg();
1265 unsigned Src1SubReg = Src1->getSubReg();
1266 Src0->setReg(Src1Reg);
1267 Src0->setSubReg(Src1SubReg);
1268 Src0->setIsKill(Src1->isKill());
1270 if (Opc == AMDGPU::V_MAC_F32_e64) {
1271 UseMI.untieRegOperand(
1272 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
1275 Src1->ChangeToImmediate(Imm);
1277 removeModOperands(UseMI);
1278 UseMI.setDesc(get(AMDGPU::V_MADMK_F32));
1280 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
1282 DefMI.eraseFromParent();
1287 // Added part is the constant: Use v_madak_f32
1288 if (Src2->isReg() && Src2->getReg() == Reg) {
1289 // Not allowed to use constant bus for another operand.
1290 // We can however allow an inline immediate as src0.
1291 if (!Src0->isImm() &&
1292 (Src0->isReg() && RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))
1295 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
1298 const int64_t Imm = DefMI.getOperand(1).getImm();
1300 // FIXME: This would be a lot easier if we could return a new instruction
1301 // instead of having to modify in place.
1303 // Remove these first since they are at the end.
1304 UseMI.RemoveOperand(
1305 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
1306 UseMI.RemoveOperand(
1307 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
1309 if (Opc == AMDGPU::V_MAC_F32_e64) {
1310 UseMI.untieRegOperand(
1311 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
1314 // ChangingToImmediate adds Src2 back to the instruction.
1315 Src2->ChangeToImmediate(Imm);
1317 // These come before src2.
1318 removeModOperands(UseMI);
1319 UseMI.setDesc(get(AMDGPU::V_MADAK_F32));
1321 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
1323 DefMI.eraseFromParent();
1332 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
1333 int WidthB, int OffsetB) {
1334 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1335 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1336 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1337 return LowOffset + LowWidth <= HighOffset;
1340 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr &MIa,
1341 MachineInstr &MIb) const {
1342 unsigned BaseReg0, BaseReg1;
1343 int64_t Offset0, Offset1;
1345 if (getMemOpBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) &&
1346 getMemOpBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) {
1348 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
1349 // FIXME: Handle ds_read2 / ds_write2.
1352 unsigned Width0 = (*MIa.memoperands_begin())->getSize();
1353 unsigned Width1 = (*MIb.memoperands_begin())->getSize();
1354 if (BaseReg0 == BaseReg1 &&
1355 offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
1363 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr &MIa,
1365 AliasAnalysis *AA) const {
1366 assert((MIa.mayLoad() || MIa.mayStore()) &&
1367 "MIa must load from or modify a memory location");
1368 assert((MIb.mayLoad() || MIb.mayStore()) &&
1369 "MIb must load from or modify a memory location");
1371 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
1374 // XXX - Can we relax this between address spaces?
1375 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1378 // TODO: Should we check the address space from the MachineMemOperand? That
1379 // would allow us to distinguish objects we know don't alias based on the
1380 // underlying address space, even if it was lowered to a different one,
1381 // e.g. private accesses lowered to use MUBUF instructions on a scratch
1385 return checkInstOffsetsDoNotOverlap(MIa, MIb);
1387 return !isFLAT(MIb);
1390 if (isMUBUF(MIa) || isMTBUF(MIa)) {
1391 if (isMUBUF(MIb) || isMTBUF(MIb))
1392 return checkInstOffsetsDoNotOverlap(MIa, MIb);
1394 return !isFLAT(MIb) && !isSMRD(MIb);
1399 return checkInstOffsetsDoNotOverlap(MIa, MIb);
1401 return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
1406 return checkInstOffsetsDoNotOverlap(MIa, MIb);
1414 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
1416 LiveVariables *LV) const {
1418 switch (MI.getOpcode()) {
1421 case AMDGPU::V_MAC_F32_e64:
1423 case AMDGPU::V_MAC_F32_e32: {
1424 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
1425 if (Src0->isImm() && !isInlineConstant(*Src0, 4))
1431 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
1432 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
1433 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
1434 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
1436 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(AMDGPU::V_MAD_F32))
1438 .addImm(0) // Src0 mods
1440 .addImm(0) // Src1 mods
1442 .addImm(0) // Src mods
1448 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1449 const MachineBasicBlock *MBB,
1450 const MachineFunction &MF) const {
1451 // XXX - Do we want the SP check in the base implementation?
1453 // Target-independent instructions do not have an implicit-use of EXEC, even
1454 // when they operate on VGPRs. Treating EXEC modifications as scheduling
1455 // boundaries prevents incorrect movements of such instructions.
1456 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
1457 MI.modifiesRegister(AMDGPU::EXEC, &RI);
1460 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
1461 int64_t SVal = Imm.getSExtValue();
1462 if (SVal >= -16 && SVal <= 64)
1465 if (Imm.getBitWidth() == 64) {
1466 uint64_t Val = Imm.getZExtValue();
1467 return (DoubleToBits(0.0) == Val) ||
1468 (DoubleToBits(1.0) == Val) ||
1469 (DoubleToBits(-1.0) == Val) ||
1470 (DoubleToBits(0.5) == Val) ||
1471 (DoubleToBits(-0.5) == Val) ||
1472 (DoubleToBits(2.0) == Val) ||
1473 (DoubleToBits(-2.0) == Val) ||
1474 (DoubleToBits(4.0) == Val) ||
1475 (DoubleToBits(-4.0) == Val);
1478 // The actual type of the operand does not seem to matter as long
1479 // as the bits match one of the inline immediate values. For example:
1481 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
1482 // so it is a legal inline immediate.
1484 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
1485 // floating-point, so it is a legal inline immediate.
1486 uint32_t Val = Imm.getZExtValue();
1488 return (FloatToBits(0.0f) == Val) ||
1489 (FloatToBits(1.0f) == Val) ||
1490 (FloatToBits(-1.0f) == Val) ||
1491 (FloatToBits(0.5f) == Val) ||
1492 (FloatToBits(-0.5f) == Val) ||
1493 (FloatToBits(2.0f) == Val) ||
1494 (FloatToBits(-2.0f) == Val) ||
1495 (FloatToBits(4.0f) == Val) ||
1496 (FloatToBits(-4.0f) == Val);
1499 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
1500 unsigned OpSize) const {
1502 // MachineOperand provides no way to tell the true operand size, since it
1503 // only records a 64-bit value. We need to know the size to determine if a
1504 // 32-bit floating point immediate bit pattern is legal for an integer
1505 // immediate. It would be for any 32-bit integer operand, but would not be
1506 // for a 64-bit one.
1508 unsigned BitSize = 8 * OpSize;
1509 return isInlineConstant(APInt(BitSize, MO.getImm(), true));
1515 bool SIInstrInfo::isLiteralConstant(const MachineOperand &MO,
1516 unsigned OpSize) const {
1517 return MO.isImm() && !isInlineConstant(MO, OpSize);
1520 static bool compareMachineOp(const MachineOperand &Op0,
1521 const MachineOperand &Op1) {
1522 if (Op0.getType() != Op1.getType())
1525 switch (Op0.getType()) {
1526 case MachineOperand::MO_Register:
1527 return Op0.getReg() == Op1.getReg();
1528 case MachineOperand::MO_Immediate:
1529 return Op0.getImm() == Op1.getImm();
1531 llvm_unreachable("Didn't expect to be comparing these operand types");
1535 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1536 const MachineOperand &MO) const {
1537 const MCOperandInfo &OpInfo = get(MI.getOpcode()).OpInfo[OpNo];
1539 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
1541 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
1544 if (OpInfo.RegClass < 0)
1547 unsigned OpSize = RI.getRegClass(OpInfo.RegClass)->getSize();
1548 if (isLiteralConstant(MO, OpSize))
1549 return RI.opCanUseLiteralConstant(OpInfo.OperandType);
1551 return RI.opCanUseInlineConstant(OpInfo.OperandType);
1554 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
1555 int Op32 = AMDGPU::getVOPe32(Opcode);
1559 return pseudoToMCOpcode(Op32) != -1;
1562 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
1563 // The src0_modifier operand is present on all instructions
1564 // that have modifiers.
1566 return AMDGPU::getNamedOperandIdx(Opcode,
1567 AMDGPU::OpName::src0_modifiers) != -1;
1570 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
1571 unsigned OpName) const {
1572 const MachineOperand *Mods = getNamedOperand(MI, OpName);
1573 return Mods && Mods->getImm();
1576 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
1577 const MachineOperand &MO,
1578 unsigned OpSize) const {
1579 // Literal constants use the constant bus.
1580 if (isLiteralConstant(MO, OpSize))
1583 if (!MO.isReg() || !MO.isUse())
1586 if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
1587 return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
1589 // FLAT_SCR is just an SGPR pair.
1590 if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR))
1593 // EXEC register uses the constant bus.
1594 if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC)
1597 // SGPRs use the constant bus
1598 return (MO.getReg() == AMDGPU::VCC || MO.getReg() == AMDGPU::M0 ||
1599 (!MO.isImplicit() &&
1600 (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) ||
1601 AMDGPU::SGPR_64RegClass.contains(MO.getReg()))));
1604 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
1605 for (const MachineOperand &MO : MI.implicit_operands()) {
1606 // We only care about reads.
1610 switch (MO.getReg()) {
1613 case AMDGPU::FLAT_SCR:
1621 return AMDGPU::NoRegister;
1624 static bool shouldReadExec(const MachineInstr &MI) {
1625 if (SIInstrInfo::isVALU(MI)) {
1626 switch (MI.getOpcode()) {
1627 case AMDGPU::V_READLANE_B32:
1628 case AMDGPU::V_READLANE_B32_si:
1629 case AMDGPU::V_READLANE_B32_vi:
1630 case AMDGPU::V_WRITELANE_B32:
1631 case AMDGPU::V_WRITELANE_B32_si:
1632 case AMDGPU::V_WRITELANE_B32_vi:
1639 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
1640 SIInstrInfo::isSALU(MI) ||
1641 SIInstrInfo::isSMRD(MI))
1647 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
1648 StringRef &ErrInfo) const {
1649 uint16_t Opcode = MI.getOpcode();
1650 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1651 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
1652 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
1653 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
1655 // Make sure the number of operands is correct.
1656 const MCInstrDesc &Desc = get(Opcode);
1657 if (!Desc.isVariadic() &&
1658 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
1659 ErrInfo = "Instruction has wrong number of operands.";
1663 // Make sure the register classes are correct.
1664 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
1665 if (MI.getOperand(i).isFPImm()) {
1666 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
1667 "all fp values to integers.";
1671 int RegClass = Desc.OpInfo[i].RegClass;
1673 switch (Desc.OpInfo[i].OperandType) {
1674 case MCOI::OPERAND_REGISTER:
1675 if (MI.getOperand(i).isImm()) {
1676 ErrInfo = "Illegal immediate value for operand.";
1680 case AMDGPU::OPERAND_REG_IMM32:
1682 case AMDGPU::OPERAND_REG_INLINE_C:
1683 if (isLiteralConstant(MI.getOperand(i),
1684 RI.getRegClass(RegClass)->getSize())) {
1685 ErrInfo = "Illegal immediate value for operand.";
1689 case MCOI::OPERAND_IMMEDIATE:
1690 case AMDGPU::OPERAND_KIMM32:
1691 // Check if this operand is an immediate.
1692 // FrameIndex operands will be replaced by immediates, so they are
1694 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
1695 ErrInfo = "Expected immediate, but got non-immediate";
1703 if (!MI.getOperand(i).isReg())
1706 if (RegClass != -1) {
1707 unsigned Reg = MI.getOperand(i).getReg();
1708 if (Reg == AMDGPU::NoRegister ||
1709 TargetRegisterInfo::isVirtualRegister(Reg))
1712 const TargetRegisterClass *RC = RI.getRegClass(RegClass);
1713 if (!RC->contains(Reg)) {
1714 ErrInfo = "Operand has incorrect register class.";
1721 if (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI)) {
1722 // Only look at the true operands. Only a real operand can use the constant
1723 // bus, and we don't want to check pseudo-operands like the source modifier
1725 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
1727 unsigned ConstantBusCount = 0;
1729 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
1732 unsigned SGPRUsed = findImplicitSGPRRead(MI);
1733 if (SGPRUsed != AMDGPU::NoRegister)
1736 for (int OpIdx : OpIndices) {
1739 const MachineOperand &MO = MI.getOperand(OpIdx);
1740 if (usesConstantBus(MRI, MO, getOpSize(Opcode, OpIdx))) {
1742 if (MO.getReg() != SGPRUsed)
1744 SGPRUsed = MO.getReg();
1750 if (ConstantBusCount > 1) {
1751 ErrInfo = "VOP* instruction uses the constant bus more than once";
1756 // Verify misc. restrictions on specific instructions.
1757 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
1758 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
1759 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
1760 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
1761 const MachineOperand &Src2 = MI.getOperand(Src2Idx);
1762 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
1763 if (!compareMachineOp(Src0, Src1) &&
1764 !compareMachineOp(Src0, Src2)) {
1765 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
1771 // Make sure we aren't losing exec uses in the td files. This mostly requires
1772 // being careful when using let Uses to try to add other use registers.
1773 if (shouldReadExec(MI)) {
1774 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
1775 ErrInfo = "VALU instruction does not implicitly read exec mask";
1783 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) {
1784 switch (MI.getOpcode()) {
1785 default: return AMDGPU::INSTRUCTION_LIST_END;
1786 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
1787 case AMDGPU::COPY: return AMDGPU::COPY;
1788 case AMDGPU::PHI: return AMDGPU::PHI;
1789 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
1790 case AMDGPU::S_MOV_B32:
1791 return MI.getOperand(1).isReg() ?
1792 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
1793 case AMDGPU::S_ADD_I32:
1794 case AMDGPU::S_ADD_U32: return AMDGPU::V_ADD_I32_e32;
1795 case AMDGPU::S_ADDC_U32: return AMDGPU::V_ADDC_U32_e32;
1796 case AMDGPU::S_SUB_I32:
1797 case AMDGPU::S_SUB_U32: return AMDGPU::V_SUB_I32_e32;
1798 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
1799 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32;
1800 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e32;
1801 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e32;
1802 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e32;
1803 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e32;
1804 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e32;
1805 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e32;
1806 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e32;
1807 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
1808 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
1809 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
1810 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
1811 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
1812 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
1813 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
1814 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
1815 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
1816 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
1817 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
1818 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
1819 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
1820 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
1821 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
1822 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
1823 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
1824 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
1825 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
1826 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
1827 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
1828 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
1829 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
1830 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
1831 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
1832 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
1833 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
1834 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
1835 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
1836 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
1837 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
1838 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
1842 bool SIInstrInfo::isSALUOpSupportedOnVALU(const MachineInstr &MI) const {
1843 return getVALUOp(MI) != AMDGPU::INSTRUCTION_LIST_END;
1846 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
1847 unsigned OpNo) const {
1848 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1849 const MCInstrDesc &Desc = get(MI.getOpcode());
1850 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
1851 Desc.OpInfo[OpNo].RegClass == -1) {
1852 unsigned Reg = MI.getOperand(OpNo).getReg();
1854 if (TargetRegisterInfo::isVirtualRegister(Reg))
1855 return MRI.getRegClass(Reg);
1856 return RI.getPhysRegClass(Reg);
1859 unsigned RCID = Desc.OpInfo[OpNo].RegClass;
1860 return RI.getRegClass(RCID);
1863 bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const {
1864 switch (MI.getOpcode()) {
1866 case AMDGPU::REG_SEQUENCE:
1868 case AMDGPU::INSERT_SUBREG:
1869 return RI.hasVGPRs(getOpRegClass(MI, 0));
1871 return RI.hasVGPRs(getOpRegClass(MI, OpNo));
1875 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
1876 MachineBasicBlock::iterator I = MI;
1877 MachineBasicBlock *MBB = MI.getParent();
1878 MachineOperand &MO = MI.getOperand(OpIdx);
1879 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1880 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
1881 const TargetRegisterClass *RC = RI.getRegClass(RCID);
1882 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1884 Opcode = AMDGPU::COPY;
1885 else if (RI.isSGPRClass(RC))
1886 Opcode = AMDGPU::S_MOV_B32;
1888 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
1889 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
1890 VRC = &AMDGPU::VReg_64RegClass;
1892 VRC = &AMDGPU::VGPR_32RegClass;
1894 unsigned Reg = MRI.createVirtualRegister(VRC);
1895 DebugLoc DL = MBB->findDebugLoc(I);
1896 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).addOperand(MO);
1897 MO.ChangeToRegister(Reg, false);
1900 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI,
1901 MachineRegisterInfo &MRI,
1902 MachineOperand &SuperReg,
1903 const TargetRegisterClass *SuperRC,
1905 const TargetRegisterClass *SubRC)
1907 MachineBasicBlock *MBB = MI->getParent();
1908 DebugLoc DL = MI->getDebugLoc();
1909 unsigned SubReg = MRI.createVirtualRegister(SubRC);
1911 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
1912 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
1913 .addReg(SuperReg.getReg(), 0, SubIdx);
1917 // Just in case the super register is itself a sub-register, copy it to a new
1918 // value so we don't need to worry about merging its subreg index with the
1919 // SubIdx passed to this function. The register coalescer should be able to
1920 // eliminate this extra copy.
1921 unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
1923 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
1924 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
1926 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
1927 .addReg(NewSuperReg, 0, SubIdx);
1932 MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
1933 MachineBasicBlock::iterator MII,
1934 MachineRegisterInfo &MRI,
1936 const TargetRegisterClass *SuperRC,
1938 const TargetRegisterClass *SubRC) const {
1940 // XXX - Is there a better way to do this?
1941 if (SubIdx == AMDGPU::sub0)
1942 return MachineOperand::CreateImm(Op.getImm() & 0xFFFFFFFF);
1943 if (SubIdx == AMDGPU::sub1)
1944 return MachineOperand::CreateImm(Op.getImm() >> 32);
1946 llvm_unreachable("Unhandled register index for immediate");
1949 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
1951 return MachineOperand::CreateReg(SubReg, false);
1954 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
1955 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
1956 assert(Inst.getNumExplicitOperands() == 3);
1957 MachineOperand Op1 = Inst.getOperand(1);
1958 Inst.RemoveOperand(1);
1959 Inst.addOperand(Op1);
1962 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
1963 const MCOperandInfo &OpInfo,
1964 const MachineOperand &MO) const {
1968 unsigned Reg = MO.getReg();
1969 const TargetRegisterClass *RC =
1970 TargetRegisterInfo::isVirtualRegister(Reg) ?
1971 MRI.getRegClass(Reg) :
1972 RI.getPhysRegClass(Reg);
1974 const SIRegisterInfo *TRI =
1975 static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
1976 RC = TRI->getSubRegClass(RC, MO.getSubReg());
1978 // In order to be legal, the common sub-class must be equal to the
1979 // class of the current operand. For example:
1981 // v_mov_b32 s0 ; Operand defined as vsrc_32
1982 // ; RI.getCommonSubClass(s0,vsrc_32) = sgpr ; LEGAL
1984 // s_sendmsg 0, s0 ; Operand defined as m0reg
1985 // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
1987 return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
1990 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
1991 const MCOperandInfo &OpInfo,
1992 const MachineOperand &MO) const {
1994 return isLegalRegOperand(MRI, OpInfo, MO);
1996 // Handle non-register types that are treated like immediates.
1997 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
2001 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
2002 const MachineOperand *MO) const {
2003 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2004 const MCInstrDesc &InstDesc = MI.getDesc();
2005 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
2006 const TargetRegisterClass *DefinedRC =
2007 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
2009 MO = &MI.getOperand(OpIdx);
2011 if (isVALU(MI) && usesConstantBus(MRI, *MO, DefinedRC->getSize())) {
2013 RegSubRegPair SGPRUsed;
2015 SGPRUsed = RegSubRegPair(MO->getReg(), MO->getSubReg());
2017 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
2020 const MachineOperand &Op = MI.getOperand(i);
2022 if ((Op.getReg() != SGPRUsed.Reg || Op.getSubReg() != SGPRUsed.SubReg) &&
2023 usesConstantBus(MRI, Op, getOpSize(MI, i))) {
2026 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
2034 return isLegalRegOperand(MRI, OpInfo, *MO);
2037 // Handle non-register types that are treated like immediates.
2038 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI());
2041 // This operand expects an immediate.
2045 return isImmOperandLegal(MI, OpIdx, *MO);
2048 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
2049 MachineInstr &MI) const {
2050 unsigned Opc = MI.getOpcode();
2051 const MCInstrDesc &InstrDesc = get(Opc);
2053 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
2054 MachineOperand &Src1 = MI.getOperand(Src1Idx);
2056 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
2057 // we need to only have one constant bus use.
2059 // Note we do not need to worry about literal constants here. They are
2060 // disabled for the operand type for instructions because they will always
2061 // violate the one constant bus use rule.
2062 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
2063 if (HasImplicitSGPR) {
2064 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2065 MachineOperand &Src0 = MI.getOperand(Src0Idx);
2067 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg()))
2068 legalizeOpWithMove(MI, Src0Idx);
2071 // VOP2 src0 instructions support all operand types, so we don't need to check
2072 // their legality. If src1 is already legal, we don't need to do anything.
2073 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
2076 // We do not use commuteInstruction here because it is too aggressive and will
2077 // commute if it is possible. We only want to commute here if it improves
2078 // legality. This can be called a fairly large number of times so don't waste
2079 // compile time pointlessly swapping and checking legality again.
2080 if (HasImplicitSGPR || !MI.isCommutable()) {
2081 legalizeOpWithMove(MI, Src1Idx);
2085 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2086 MachineOperand &Src0 = MI.getOperand(Src0Idx);
2088 // If src0 can be used as src1, commuting will make the operands legal.
2089 // Otherwise we have to give up and insert a move.
2091 // TODO: Other immediate-like operand kinds could be commuted if there was a
2092 // MachineOperand::ChangeTo* for them.
2093 if ((!Src1.isImm() && !Src1.isReg()) ||
2094 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
2095 legalizeOpWithMove(MI, Src1Idx);
2099 int CommutedOpc = commuteOpcode(MI);
2100 if (CommutedOpc == -1) {
2101 legalizeOpWithMove(MI, Src1Idx);
2105 MI.setDesc(get(CommutedOpc));
2107 unsigned Src0Reg = Src0.getReg();
2108 unsigned Src0SubReg = Src0.getSubReg();
2109 bool Src0Kill = Src0.isKill();
2112 Src0.ChangeToImmediate(Src1.getImm());
2113 else if (Src1.isReg()) {
2114 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
2115 Src0.setSubReg(Src1.getSubReg());
2117 llvm_unreachable("Should only have register or immediate operands");
2119 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
2120 Src1.setSubReg(Src0SubReg);
2123 // Legalize VOP3 operands. Because all operand types are supported for any
2124 // operand, and since literal constants are not allowed and should never be
2125 // seen, we only need to worry about inserting copies if we use multiple SGPR
2127 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
2128 MachineInstr &MI) const {
2129 unsigned Opc = MI.getOpcode();
2132 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
2133 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
2134 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
2137 // Find the one SGPR operand we are allowed to use.
2138 unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
2140 for (unsigned i = 0; i < 3; ++i) {
2141 int Idx = VOP3Idx[i];
2144 MachineOperand &MO = MI.getOperand(Idx);
2146 // We should never see a VOP3 instruction with an illegal immediate operand.
2150 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
2151 continue; // VGPRs are legal
2153 if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) {
2154 SGPRReg = MO.getReg();
2155 // We can use one SGPR in each VOP3 instruction.
2159 // If we make it this far, then the operand is not legal and we must
2161 legalizeOpWithMove(MI, Idx);
2165 unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
2166 MachineRegisterInfo &MRI) const {
2167 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
2168 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
2169 unsigned DstReg = MRI.createVirtualRegister(SRC);
2170 unsigned SubRegs = VRC->getSize() / 4;
2172 SmallVector<unsigned, 8> SRegs;
2173 for (unsigned i = 0; i < SubRegs; ++i) {
2174 unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
2175 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
2176 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
2177 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
2178 SRegs.push_back(SGPR);
2181 MachineInstrBuilder MIB =
2182 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
2183 get(AMDGPU::REG_SEQUENCE), DstReg);
2184 for (unsigned i = 0; i < SubRegs; ++i) {
2185 MIB.addReg(SRegs[i]);
2186 MIB.addImm(RI.getSubRegFromChannel(i));
2191 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
2192 MachineInstr &MI) const {
2194 // If the pointer is store in VGPRs, then we need to move them to
2195 // SGPRs using v_readfirstlane. This is safe because we only select
2196 // loads with uniform pointers to SMRD instruction so we know the
2197 // pointer value is uniform.
2198 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
2199 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
2200 unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
2201 SBase->setReg(SGPR);
2205 void SIInstrInfo::legalizeOperands(MachineInstr &MI) const {
2206 MachineFunction &MF = *MI.getParent()->getParent();
2207 MachineRegisterInfo &MRI = MF.getRegInfo();
2210 if (isVOP2(MI) || isVOPC(MI)) {
2211 legalizeOperandsVOP2(MRI, MI);
2217 legalizeOperandsVOP3(MRI, MI);
2223 legalizeOperandsSMRD(MRI, MI);
2227 // Legalize REG_SEQUENCE and PHI
2228 // The register class of the operands much be the same type as the register
2229 // class of the output.
2230 if (MI.getOpcode() == AMDGPU::PHI) {
2231 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
2232 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
2233 if (!MI.getOperand(i).isReg() ||
2234 !TargetRegisterInfo::isVirtualRegister(MI.getOperand(i).getReg()))
2236 const TargetRegisterClass *OpRC =
2237 MRI.getRegClass(MI.getOperand(i).getReg());
2238 if (RI.hasVGPRs(OpRC)) {
2245 // If any of the operands are VGPR registers, then they all most be
2246 // otherwise we will create illegal VGPR->SGPR copies when legalizing
2248 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
2251 VRC = RI.getEquivalentVGPRClass(SRC);
2258 // Update all the operands so they have the same type.
2259 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2260 MachineOperand &Op = MI.getOperand(I);
2261 if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg()))
2263 unsigned DstReg = MRI.createVirtualRegister(RC);
2265 // MI is a PHI instruction.
2266 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
2267 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
2269 BuildMI(*InsertBB, Insert, MI.getDebugLoc(), get(AMDGPU::COPY), DstReg)
2275 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
2276 // VGPR dest type and SGPR sources, insert copies so all operands are
2277 // VGPRs. This seems to help operand folding / the register coalescer.
2278 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
2279 MachineBasicBlock *MBB = MI.getParent();
2280 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
2281 if (RI.hasVGPRs(DstRC)) {
2282 // Update all the operands so they are VGPR register classes. These may
2283 // not be the same register class because REG_SEQUENCE supports mixing
2284 // subregister index types e.g. sub0_sub1 + sub2 + sub3
2285 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2286 MachineOperand &Op = MI.getOperand(I);
2287 if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg()))
2290 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
2291 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
2295 unsigned DstReg = MRI.createVirtualRegister(VRC);
2297 BuildMI(*MBB, MI, MI.getDebugLoc(), get(AMDGPU::COPY), DstReg)
2308 // Legalize INSERT_SUBREG
2309 // src0 must have the same register class as dst
2310 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
2311 unsigned Dst = MI.getOperand(0).getReg();
2312 unsigned Src0 = MI.getOperand(1).getReg();
2313 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
2314 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
2315 if (DstRC != Src0RC) {
2316 MachineBasicBlock &MBB = *MI.getParent();
2317 unsigned NewSrc0 = MRI.createVirtualRegister(DstRC);
2318 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::COPY), NewSrc0)
2320 MI.getOperand(1).setReg(NewSrc0);
2325 // Legalize MIMG and MUBUF/MTBUF for shaders.
2327 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
2328 // scratch memory access. In both cases, the legalization never involves
2329 // conversion to the addr64 form.
2331 (AMDGPU::isShader(MF.getFunction()->getCallingConv()) &&
2332 (isMUBUF(MI) || isMTBUF(MI)))) {
2333 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
2334 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
2335 unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
2336 SRsrc->setReg(SGPR);
2339 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
2340 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
2341 unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
2342 SSamp->setReg(SGPR);
2347 // Legalize MUBUF* instructions by converting to addr64 form.
2348 // FIXME: If we start using the non-addr64 instructions for compute, we
2349 // may need to legalize them as above. This especially applies to the
2350 // buffer_load_format_* variants and variants with idxen (or bothen).
2352 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
2353 if (SRsrcIdx != -1) {
2354 // We have an MUBUF instruction
2355 MachineOperand *SRsrc = &MI.getOperand(SRsrcIdx);
2356 unsigned SRsrcRC = get(MI.getOpcode()).OpInfo[SRsrcIdx].RegClass;
2357 if (RI.getCommonSubClass(MRI.getRegClass(SRsrc->getReg()),
2358 RI.getRegClass(SRsrcRC))) {
2359 // The operands are legal.
2360 // FIXME: We may need to legalize operands besided srsrc.
2364 MachineBasicBlock &MBB = *MI.getParent();
2366 // Extract the ptr from the resource descriptor.
2367 unsigned SRsrcPtr = buildExtractSubReg(MI, MRI, *SRsrc,
2368 &AMDGPU::VReg_128RegClass, AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
2370 // Create an empty resource descriptor
2371 unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
2372 unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
2373 unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
2374 unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
2375 uint64_t RsrcDataFormat = getDefaultRsrcDataFormat();
2378 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::S_MOV_B64), Zero64)
2381 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
2382 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
2383 .addImm(RsrcDataFormat & 0xFFFFFFFF);
2385 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
2386 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
2387 .addImm(RsrcDataFormat >> 32);
2389 // NewSRsrc = {Zero64, SRsrcFormat}
2390 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewSRsrc)
2392 .addImm(AMDGPU::sub0_sub1)
2393 .addReg(SRsrcFormatLo)
2394 .addImm(AMDGPU::sub2)
2395 .addReg(SRsrcFormatHi)
2396 .addImm(AMDGPU::sub3);
2398 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
2399 unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
2401 // This is already an ADDR64 instruction so we need to add the pointer
2402 // extracted from the resource descriptor to the current value of VAddr.
2403 unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2404 unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2406 // NewVaddrLo = SRsrcPtr:sub0 + VAddr:sub0
2407 DebugLoc DL = MI.getDebugLoc();
2408 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo)
2409 .addReg(SRsrcPtr, 0, AMDGPU::sub0)
2410 .addReg(VAddr->getReg(), 0, AMDGPU::sub0);
2412 // NewVaddrHi = SRsrcPtr:sub1 + VAddr:sub1
2413 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi)
2414 .addReg(SRsrcPtr, 0, AMDGPU::sub1)
2415 .addReg(VAddr->getReg(), 0, AMDGPU::sub1);
2417 // NewVaddr = {NewVaddrHi, NewVaddrLo}
2418 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
2420 .addImm(AMDGPU::sub0)
2422 .addImm(AMDGPU::sub1);
2424 // This instructions is the _OFFSET variant, so we need to convert it to
2426 assert(MBB.getParent()->getSubtarget<SISubtarget>().getGeneration()
2427 < SISubtarget::VOLCANIC_ISLANDS &&
2428 "FIXME: Need to emit flat atomics here");
2430 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
2431 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
2432 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
2433 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
2435 // Atomics rith return have have an additional tied operand and are
2436 // missing some of the special bits.
2437 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
2438 MachineInstr *Addr64;
2441 // Regular buffer load / store.
2442 MachineInstrBuilder MIB =
2443 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
2445 .addReg(AMDGPU::NoRegister) // Dummy value for vaddr.
2446 // This will be replaced later
2447 // with the new value of vaddr.
2449 .addOperand(*SOffset)
2450 .addOperand(*Offset);
2452 // Atomics do not have this operand.
2453 if (const MachineOperand *GLC =
2454 getNamedOperand(MI, AMDGPU::OpName::glc)) {
2455 MIB.addImm(GLC->getImm());
2458 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
2460 if (const MachineOperand *TFE =
2461 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
2462 MIB.addImm(TFE->getImm());
2465 MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
2468 // Atomics with return.
2469 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
2471 .addOperand(*VDataIn)
2472 .addReg(AMDGPU::NoRegister) // Dummy value for vaddr.
2473 // This will be replaced later
2474 // with the new value of vaddr.
2476 .addOperand(*SOffset)
2477 .addOperand(*Offset)
2478 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
2479 .setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
2482 MI.removeFromParent();
2484 // NewVaddr = {NewVaddrHi, NewVaddrLo}
2485 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
2487 .addReg(SRsrcPtr, 0, AMDGPU::sub0)
2488 .addImm(AMDGPU::sub0)
2489 .addReg(SRsrcPtr, 0, AMDGPU::sub1)
2490 .addImm(AMDGPU::sub1);
2492 VAddr = getNamedOperand(*Addr64, AMDGPU::OpName::vaddr);
2493 SRsrc = getNamedOperand(*Addr64, AMDGPU::OpName::srsrc);
2496 // Update the instruction to use NewVaddr
2497 VAddr->setReg(NewVAddr);
2498 // Update the instruction to use NewSRsrc
2499 SRsrc->setReg(NewSRsrc);
2503 void SIInstrInfo::moveToVALU(MachineInstr &TopInst) const {
2504 SmallVector<MachineInstr *, 128> Worklist;
2505 Worklist.push_back(&TopInst);
2507 while (!Worklist.empty()) {
2508 MachineInstr &Inst = *Worklist.pop_back_val();
2509 MachineBasicBlock *MBB = Inst.getParent();
2510 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2512 unsigned Opcode = Inst.getOpcode();
2513 unsigned NewOpcode = getVALUOp(Inst);
2515 // Handle some special cases
2519 case AMDGPU::S_AND_B64:
2520 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_AND_B32_e64);
2521 Inst.eraseFromParent();
2524 case AMDGPU::S_OR_B64:
2525 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_OR_B32_e64);
2526 Inst.eraseFromParent();
2529 case AMDGPU::S_XOR_B64:
2530 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_XOR_B32_e64);
2531 Inst.eraseFromParent();
2534 case AMDGPU::S_NOT_B64:
2535 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::V_NOT_B32_e32);
2536 Inst.eraseFromParent();
2539 case AMDGPU::S_BCNT1_I32_B64:
2540 splitScalar64BitBCNT(Worklist, Inst);
2541 Inst.eraseFromParent();
2544 case AMDGPU::S_BFE_I64: {
2545 splitScalar64BitBFE(Worklist, Inst);
2546 Inst.eraseFromParent();
2550 case AMDGPU::S_LSHL_B32:
2551 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
2552 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
2556 case AMDGPU::S_ASHR_I32:
2557 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
2558 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
2562 case AMDGPU::S_LSHR_B32:
2563 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
2564 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
2568 case AMDGPU::S_LSHL_B64:
2569 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
2570 NewOpcode = AMDGPU::V_LSHLREV_B64;
2574 case AMDGPU::S_ASHR_I64:
2575 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
2576 NewOpcode = AMDGPU::V_ASHRREV_I64;
2580 case AMDGPU::S_LSHR_B64:
2581 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
2582 NewOpcode = AMDGPU::V_LSHRREV_B64;
2587 case AMDGPU::S_ABS_I32:
2588 lowerScalarAbs(Worklist, Inst);
2589 Inst.eraseFromParent();
2592 case AMDGPU::S_CBRANCH_SCC0:
2593 case AMDGPU::S_CBRANCH_SCC1:
2594 // Clear unused bits of vcc
2595 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
2597 .addReg(AMDGPU::EXEC)
2598 .addReg(AMDGPU::VCC);
2601 case AMDGPU::S_BFE_U64:
2602 case AMDGPU::S_BFM_B64:
2603 llvm_unreachable("Moving this op to VALU not implemented");
2606 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
2607 // We cannot move this instruction to the VALU, so we should try to
2608 // legalize its operands instead.
2609 legalizeOperands(Inst);
2613 // Use the new VALU Opcode.
2614 const MCInstrDesc &NewDesc = get(NewOpcode);
2615 Inst.setDesc(NewDesc);
2617 // Remove any references to SCC. Vector instructions can't read from it, and
2618 // We're just about to add the implicit use / defs of VCC, and we don't want
2620 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
2621 MachineOperand &Op = Inst.getOperand(i);
2622 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
2623 Inst.RemoveOperand(i);
2624 addSCCDefUsersToVALUWorklist(Inst, Worklist);
2628 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
2629 // We are converting these to a BFE, so we need to add the missing
2630 // operands for the size and offset.
2631 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
2632 Inst.addOperand(MachineOperand::CreateImm(0));
2633 Inst.addOperand(MachineOperand::CreateImm(Size));
2635 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
2636 // The VALU version adds the second operand to the result, so insert an
2638 Inst.addOperand(MachineOperand::CreateImm(0));
2641 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent());
2643 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
2644 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
2645 // If we need to move this to VGPRs, we need to unpack the second operand
2646 // back into the 2 separate ones for bit offset and width.
2647 assert(OffsetWidthOp.isImm() &&
2648 "Scalar BFE is only implemented for constant width and offset");
2649 uint32_t Imm = OffsetWidthOp.getImm();
2651 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
2652 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
2653 Inst.RemoveOperand(2); // Remove old immediate.
2654 Inst.addOperand(MachineOperand::CreateImm(Offset));
2655 Inst.addOperand(MachineOperand::CreateImm(BitWidth));
2658 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
2659 unsigned NewDstReg = AMDGPU::NoRegister;
2661 // Update the destination register class.
2662 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
2666 unsigned DstReg = Inst.getOperand(0).getReg();
2667 NewDstReg = MRI.createVirtualRegister(NewDstRC);
2668 MRI.replaceRegWith(DstReg, NewDstReg);
2671 // Legalize the operands
2672 legalizeOperands(Inst);
2675 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
2679 void SIInstrInfo::lowerScalarAbs(SmallVectorImpl<MachineInstr *> &Worklist,
2680 MachineInstr &Inst) const {
2681 MachineBasicBlock &MBB = *Inst.getParent();
2682 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2683 MachineBasicBlock::iterator MII = Inst;
2684 DebugLoc DL = Inst.getDebugLoc();
2686 MachineOperand &Dest = Inst.getOperand(0);
2687 MachineOperand &Src = Inst.getOperand(1);
2688 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2689 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2691 BuildMI(MBB, MII, DL, get(AMDGPU::V_SUB_I32_e32), TmpReg)
2693 .addReg(Src.getReg());
2695 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
2696 .addReg(Src.getReg())
2699 MRI.replaceRegWith(Dest.getReg(), ResultReg);
2700 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
2703 void SIInstrInfo::splitScalar64BitUnaryOp(
2704 SmallVectorImpl<MachineInstr *> &Worklist, MachineInstr &Inst,
2705 unsigned Opcode) const {
2706 MachineBasicBlock &MBB = *Inst.getParent();
2707 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2709 MachineOperand &Dest = Inst.getOperand(0);
2710 MachineOperand &Src0 = Inst.getOperand(1);
2711 DebugLoc DL = Inst.getDebugLoc();
2713 MachineBasicBlock::iterator MII = Inst;
2715 const MCInstrDesc &InstDesc = get(Opcode);
2716 const TargetRegisterClass *Src0RC = Src0.isReg() ?
2717 MRI.getRegClass(Src0.getReg()) :
2718 &AMDGPU::SGPR_32RegClass;
2720 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
2722 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
2723 AMDGPU::sub0, Src0SubRC);
2725 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
2726 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
2727 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
2729 unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
2730 BuildMI(MBB, MII, DL, InstDesc, DestSub0)
2731 .addOperand(SrcReg0Sub0);
2733 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
2734 AMDGPU::sub1, Src0SubRC);
2736 unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
2737 BuildMI(MBB, MII, DL, InstDesc, DestSub1)
2738 .addOperand(SrcReg0Sub1);
2740 unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
2741 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
2743 .addImm(AMDGPU::sub0)
2745 .addImm(AMDGPU::sub1);
2747 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
2749 // We don't need to legalizeOperands here because for a single operand, src0
2750 // will support any kind of input.
2752 // Move all users of this moved value.
2753 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
2756 void SIInstrInfo::splitScalar64BitBinaryOp(
2757 SmallVectorImpl<MachineInstr *> &Worklist, MachineInstr &Inst,
2758 unsigned Opcode) const {
2759 MachineBasicBlock &MBB = *Inst.getParent();
2760 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2762 MachineOperand &Dest = Inst.getOperand(0);
2763 MachineOperand &Src0 = Inst.getOperand(1);
2764 MachineOperand &Src1 = Inst.getOperand(2);
2765 DebugLoc DL = Inst.getDebugLoc();
2767 MachineBasicBlock::iterator MII = Inst;
2769 const MCInstrDesc &InstDesc = get(Opcode);
2770 const TargetRegisterClass *Src0RC = Src0.isReg() ?
2771 MRI.getRegClass(Src0.getReg()) :
2772 &AMDGPU::SGPR_32RegClass;
2774 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
2775 const TargetRegisterClass *Src1RC = Src1.isReg() ?
2776 MRI.getRegClass(Src1.getReg()) :
2777 &AMDGPU::SGPR_32RegClass;
2779 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
2781 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
2782 AMDGPU::sub0, Src0SubRC);
2783 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
2784 AMDGPU::sub0, Src1SubRC);
2786 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
2787 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
2788 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
2790 unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
2791 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
2792 .addOperand(SrcReg0Sub0)
2793 .addOperand(SrcReg1Sub0);
2795 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
2796 AMDGPU::sub1, Src0SubRC);
2797 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
2798 AMDGPU::sub1, Src1SubRC);
2800 unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
2801 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
2802 .addOperand(SrcReg0Sub1)
2803 .addOperand(SrcReg1Sub1);
2805 unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
2806 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
2808 .addImm(AMDGPU::sub0)
2810 .addImm(AMDGPU::sub1);
2812 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
2814 // Try to legalize the operands in case we need to swap the order to keep it
2816 legalizeOperands(LoHalf);
2817 legalizeOperands(HiHalf);
2819 // Move all users of this moved vlaue.
2820 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
2823 void SIInstrInfo::splitScalar64BitBCNT(
2824 SmallVectorImpl<MachineInstr *> &Worklist, MachineInstr &Inst) const {
2825 MachineBasicBlock &MBB = *Inst.getParent();
2826 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2828 MachineBasicBlock::iterator MII = Inst;
2829 DebugLoc DL = Inst.getDebugLoc();
2831 MachineOperand &Dest = Inst.getOperand(0);
2832 MachineOperand &Src = Inst.getOperand(1);
2834 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
2835 const TargetRegisterClass *SrcRC = Src.isReg() ?
2836 MRI.getRegClass(Src.getReg()) :
2837 &AMDGPU::SGPR_32RegClass;
2839 unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2840 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2842 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
2844 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
2845 AMDGPU::sub0, SrcSubRC);
2846 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
2847 AMDGPU::sub1, SrcSubRC);
2849 BuildMI(MBB, MII, DL, InstDesc, MidReg)
2850 .addOperand(SrcRegSub0)
2853 BuildMI(MBB, MII, DL, InstDesc, ResultReg)
2854 .addOperand(SrcRegSub1)
2857 MRI.replaceRegWith(Dest.getReg(), ResultReg);
2859 // We don't need to legalize operands here. src0 for etiher instruction can be
2860 // an SGPR, and the second input is unused or determined here.
2861 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
2864 void SIInstrInfo::splitScalar64BitBFE(SmallVectorImpl<MachineInstr *> &Worklist,
2865 MachineInstr &Inst) const {
2866 MachineBasicBlock &MBB = *Inst.getParent();
2867 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2868 MachineBasicBlock::iterator MII = Inst;
2869 DebugLoc DL = Inst.getDebugLoc();
2871 MachineOperand &Dest = Inst.getOperand(0);
2872 uint32_t Imm = Inst.getOperand(2).getImm();
2873 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
2874 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
2878 // Only sext_inreg cases handled.
2879 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
2880 Offset == 0 && "Not implemented");
2882 if (BitWidth < 32) {
2883 unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2884 unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2885 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
2887 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
2888 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
2892 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
2896 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
2898 .addImm(AMDGPU::sub0)
2900 .addImm(AMDGPU::sub1);
2902 MRI.replaceRegWith(Dest.getReg(), ResultReg);
2903 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
2907 MachineOperand &Src = Inst.getOperand(1);
2908 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2909 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
2911 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
2913 .addReg(Src.getReg(), 0, AMDGPU::sub0);
2915 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
2916 .addReg(Src.getReg(), 0, AMDGPU::sub0)
2917 .addImm(AMDGPU::sub0)
2919 .addImm(AMDGPU::sub1);
2921 MRI.replaceRegWith(Dest.getReg(), ResultReg);
2922 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
2925 void SIInstrInfo::addUsersToMoveToVALUWorklist(
2927 MachineRegisterInfo &MRI,
2928 SmallVectorImpl<MachineInstr *> &Worklist) const {
2929 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
2930 E = MRI.use_end(); I != E; ++I) {
2931 MachineInstr &UseMI = *I->getParent();
2932 if (!canReadVGPR(UseMI, I.getOperandNo())) {
2933 Worklist.push_back(&UseMI);
2938 void SIInstrInfo::addSCCDefUsersToVALUWorklist(
2939 MachineInstr &SCCDefInst, SmallVectorImpl<MachineInstr *> &Worklist) const {
2940 // This assumes that all the users of SCC are in the same block
2942 for (MachineInstr &MI :
2943 llvm::make_range(MachineBasicBlock::iterator(SCCDefInst),
2944 SCCDefInst.getParent()->end())) {
2945 // Exit if we find another SCC def.
2946 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC) != -1)
2949 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC) != -1)
2950 Worklist.push_back(&MI);
2954 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
2955 const MachineInstr &Inst) const {
2956 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
2958 switch (Inst.getOpcode()) {
2959 // For target instructions, getOpRegClass just returns the virtual register
2960 // class associated with the operand, so we need to find an equivalent VGPR
2961 // register class in order to move the instruction to the VALU.
2964 case AMDGPU::REG_SEQUENCE:
2965 case AMDGPU::INSERT_SUBREG:
2966 if (RI.hasVGPRs(NewDstRC))
2969 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
2978 // Find the one SGPR operand we are allowed to use.
2979 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
2980 int OpIndices[3]) const {
2981 const MCInstrDesc &Desc = MI.getDesc();
2983 // Find the one SGPR operand we are allowed to use.
2985 // First we need to consider the instruction's operand requirements before
2986 // legalizing. Some operands are required to be SGPRs, such as implicit uses
2987 // of VCC, but we are still bound by the constant bus requirement to only use
2990 // If the operand's class is an SGPR, we can never move it.
2992 unsigned SGPRReg = findImplicitSGPRRead(MI);
2993 if (SGPRReg != AMDGPU::NoRegister)
2996 unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
2997 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2999 for (unsigned i = 0; i < 3; ++i) {
3000 int Idx = OpIndices[i];
3004 const MachineOperand &MO = MI.getOperand(Idx);
3008 // Is this operand statically required to be an SGPR based on the operand
3010 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
3011 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
3015 // If this could be a VGPR or an SGPR, Check the dynamic register class.
3016 unsigned Reg = MO.getReg();
3017 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
3018 if (RI.isSGPRClass(RegRC))
3022 // We don't have a required SGPR operand, so we have a bit more freedom in
3023 // selecting operands to move.
3025 // Try to select the most used SGPR. If an SGPR is equal to one of the
3026 // others, we choose that.
3029 // V_FMA_F32 v0, s0, s0, s0 -> No moves
3030 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
3032 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
3035 if (UsedSGPRs[0] != AMDGPU::NoRegister) {
3036 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
3037 SGPRReg = UsedSGPRs[0];
3040 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
3041 if (UsedSGPRs[1] == UsedSGPRs[2])
3042 SGPRReg = UsedSGPRs[1];
3048 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
3049 unsigned OperandName) const {
3050 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
3054 return &MI.getOperand(Idx);
3057 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
3058 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
3059 if (ST.isAmdHsaOS()) {
3060 RsrcDataFormat |= (1ULL << 56);
3062 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS)
3064 RsrcDataFormat |= (2ULL << 59);
3067 return RsrcDataFormat;
3070 uint64_t SIInstrInfo::getScratchRsrcWords23() const {
3071 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
3072 AMDGPU::RSRC_TID_ENABLE |
3073 0xffffffff; // Size;
3075 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
3077 Rsrc23 |= (EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT) |
3079 (UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT);
3081 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
3082 // Clear them unless we want a huge stride.
3083 if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS)
3084 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
3089 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
3090 unsigned Opc = MI.getOpcode();
3095 bool SIInstrInfo::isHighLatencyInstruction(const MachineInstr &MI) const {
3096 unsigned Opc = MI.getOpcode();
3098 return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
3101 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
3102 unsigned Opc = MI.getOpcode();
3103 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
3104 unsigned DescSize = Desc.getSize();
3106 // If we have a definitive size, we can use it. Otherwise we need to inspect
3107 // the operands to know the size.
3108 if (DescSize == 8 || DescSize == 4)
3111 assert(DescSize == 0);
3113 // 4-byte instructions may have a 32-bit literal encoded after them. Check
3114 // operands that coud ever be literals.
3115 if (isVALU(MI) || isSALU(MI)) {
3116 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3118 return 4; // No operands.
3120 if (isLiteralConstant(MI.getOperand(Src0Idx), getOpSize(MI, Src0Idx)))
3123 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3127 if (isLiteralConstant(MI.getOperand(Src1Idx), getOpSize(MI, Src1Idx)))
3134 case TargetOpcode::IMPLICIT_DEF:
3135 case TargetOpcode::KILL:
3136 case TargetOpcode::DBG_VALUE:
3137 case TargetOpcode::BUNDLE:
3138 case TargetOpcode::EH_LABEL:
3140 case TargetOpcode::INLINEASM: {
3141 const MachineFunction *MF = MI.getParent()->getParent();
3142 const char *AsmStr = MI.getOperand(0).getSymbolName();
3143 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
3146 llvm_unreachable("unable to find instruction size");
3150 ArrayRef<std::pair<int, const char *>>
3151 SIInstrInfo::getSerializableTargetIndices() const {
3152 static const std::pair<int, const char *> TargetIndices[] = {
3153 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
3154 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
3155 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
3156 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
3157 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
3158 return makeArrayRef(TargetIndices);
3161 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
3162 /// post-RA version of misched uses CreateTargetMIHazardRecognizer.
3163 ScheduleHazardRecognizer *
3164 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
3165 const ScheduleDAG *DAG) const {
3166 return new GCNHazardRecognizer(DAG->MF);
3169 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
3171 ScheduleHazardRecognizer *
3172 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
3173 return new GCNHazardRecognizer(MF);