1 //===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// This file implements the targeting of the InstructionSelector class for
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
14 #include "AMDGPUInstructionSelector.h"
15 #include "AMDGPUInstrInfo.h"
16 #include "AMDGPURegisterBankInfo.h"
17 #include "AMDGPURegisterInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "AMDGPUTargetMachine.h"
20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25 #include "llvm/CodeGen/GlobalISel/Utils.h"
26 #include "llvm/CodeGen/MachineBasicBlock.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineInstr.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/Support/Debug.h"
33 #include "llvm/Support/raw_ostream.h"
35 #define DEBUG_TYPE "amdgpu-isel"
38 using namespace MIPatternMatch;
40 #define GET_GLOBALISEL_IMPL
41 #define AMDGPUSubtarget GCNSubtarget
42 #include "AMDGPUGenGlobalISel.inc"
43 #undef GET_GLOBALISEL_IMPL
44 #undef AMDGPUSubtarget
46 AMDGPUInstructionSelector::AMDGPUInstructionSelector(
47 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI,
48 const AMDGPUTargetMachine &TM)
49 : InstructionSelector(), TII(*STI.getInstrInfo()),
50 TRI(*STI.getRegisterInfo()), RBI(RBI), TM(TM),
52 EnableLateStructurizeCFG(AMDGPUTargetMachine::EnableLateStructurizeCFG),
53 #define GET_GLOBALISEL_PREDICATES_INIT
54 #include "AMDGPUGenGlobalISel.inc"
55 #undef GET_GLOBALISEL_PREDICATES_INIT
56 #define GET_GLOBALISEL_TEMPORARIES_INIT
57 #include "AMDGPUGenGlobalISel.inc"
58 #undef GET_GLOBALISEL_TEMPORARIES_INIT
62 const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
64 static bool isSCC(Register Reg, const MachineRegisterInfo &MRI) {
65 if (TargetRegisterInfo::isPhysicalRegister(Reg))
66 return Reg == AMDGPU::SCC;
68 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
69 const TargetRegisterClass *RC =
70 RegClassOrBank.dyn_cast<const TargetRegisterClass*>();
72 // FIXME: This is ambiguous for wave32. This could be SCC or VCC, but the
73 // context of the register bank has been lost.
74 if (RC->getID() != AMDGPU::SReg_32_XM0RegClassID)
76 const LLT Ty = MRI.getType(Reg);
77 return Ty.isValid() && Ty.getSizeInBits() == 1;
80 const RegisterBank *RB = RegClassOrBank.get<const RegisterBank *>();
81 return RB->getID() == AMDGPU::SCCRegBankID;
84 bool AMDGPUInstructionSelector::isVCC(Register Reg,
85 const MachineRegisterInfo &MRI) const {
86 if (TargetRegisterInfo::isPhysicalRegister(Reg))
87 return Reg == TRI.getVCC();
89 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
90 const TargetRegisterClass *RC =
91 RegClassOrBank.dyn_cast<const TargetRegisterClass*>();
93 const LLT Ty = MRI.getType(Reg);
94 return RC->hasSuperClassEq(TRI.getBoolRC()) &&
95 Ty.isValid() && Ty.getSizeInBits() == 1;
98 const RegisterBank *RB = RegClassOrBank.get<const RegisterBank *>();
99 return RB->getID() == AMDGPU::VCCRegBankID;
102 bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
103 const DebugLoc &DL = I.getDebugLoc();
104 MachineBasicBlock *BB = I.getParent();
105 MachineFunction *MF = BB->getParent();
106 MachineRegisterInfo &MRI = MF->getRegInfo();
107 I.setDesc(TII.get(TargetOpcode::COPY));
109 const MachineOperand &Src = I.getOperand(1);
110 MachineOperand &Dst = I.getOperand(0);
111 Register DstReg = Dst.getReg();
112 Register SrcReg = Src.getReg();
114 if (isVCC(DstReg, MRI)) {
115 if (SrcReg == AMDGPU::SCC) {
116 const TargetRegisterClass *RC
117 = TRI.getConstrainedRegClassForOperand(Dst, MRI);
120 return RBI.constrainGenericRegister(DstReg, *RC, MRI);
123 if (!isVCC(SrcReg, MRI)) {
124 // TODO: Should probably leave the copy and let copyPhysReg expand it.
125 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), MRI))
128 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg)
132 if (!MRI.getRegClassOrNull(SrcReg))
133 MRI.setRegClass(SrcReg, TRI.getConstrainedRegClassForOperand(Src, MRI));
138 const TargetRegisterClass *RC =
139 TRI.getConstrainedRegClassForOperand(Dst, MRI);
140 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, MRI))
143 // Don't constrain the source register to a class so the def instruction
144 // handles it (unless it's undef).
146 // FIXME: This is a hack. When selecting the def, we neeed to know
147 // specifically know that the result is VCCRegBank, and not just an SGPR
148 // with size 1. An SReg_32 with size 1 is ambiguous with wave32.
150 const TargetRegisterClass *SrcRC =
151 TRI.getConstrainedRegClassForOperand(Src, MRI);
152 if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI))
159 for (const MachineOperand &MO : I.operands()) {
160 if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
163 const TargetRegisterClass *RC =
164 TRI.getConstrainedRegClassForOperand(MO, MRI);
167 RBI.constrainGenericRegister(MO.getReg(), *RC, MRI);
172 bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
173 MachineBasicBlock *BB = I.getParent();
174 MachineFunction *MF = BB->getParent();
175 MachineRegisterInfo &MRI = MF->getRegInfo();
177 const Register DefReg = I.getOperand(0).getReg();
178 const LLT DefTy = MRI.getType(DefReg);
180 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
182 const RegClassOrRegBank &RegClassOrBank =
183 MRI.getRegClassOrRegBank(DefReg);
185 const TargetRegisterClass *DefRC
186 = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
188 if (!DefTy.isValid()) {
189 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
193 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
194 if (RB.getID() == AMDGPU::SCCRegBankID) {
195 LLVM_DEBUG(dbgs() << "illegal scc phi\n");
199 DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, MRI);
201 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
206 I.setDesc(TII.get(TargetOpcode::PHI));
207 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
211 AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
212 const TargetRegisterClass &SubRC,
213 unsigned SubIdx) const {
215 MachineInstr *MI = MO.getParent();
216 MachineBasicBlock *BB = MO.getParent()->getParent();
217 MachineFunction *MF = BB->getParent();
218 MachineRegisterInfo &MRI = MF->getRegInfo();
219 Register DstReg = MRI.createVirtualRegister(&SubRC);
222 unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.getSubReg(), SubIdx);
223 unsigned Reg = MO.getReg();
224 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
225 .addReg(Reg, 0, ComposedSubIdx);
227 return MachineOperand::CreateReg(DstReg, MO.isDef(), MO.isImplicit(),
228 MO.isKill(), MO.isDead(), MO.isUndef(),
229 MO.isEarlyClobber(), 0, MO.isDebug(),
230 MO.isInternalRead());
235 APInt Imm(64, MO.getImm());
239 llvm_unreachable("do not know to split immediate with this sub index.");
241 return MachineOperand::CreateImm(Imm.getLoBits(32).getSExtValue());
243 return MachineOperand::CreateImm(Imm.getHiBits(32).getSExtValue());
247 static int64_t getConstant(const MachineInstr *MI) {
248 return MI->getOperand(1).getCImm()->getSExtValue();
251 static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
254 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
256 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
258 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
260 llvm_unreachable("not a bit op");
264 bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
265 MachineBasicBlock *BB = I.getParent();
266 MachineFunction *MF = BB->getParent();
267 MachineRegisterInfo &MRI = MF->getRegInfo();
268 MachineOperand &Dst = I.getOperand(0);
269 MachineOperand &Src0 = I.getOperand(1);
270 MachineOperand &Src1 = I.getOperand(2);
271 Register DstReg = Dst.getReg();
272 unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
274 const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
275 if (DstRB->getID() == AMDGPU::VCCRegBankID) {
276 const TargetRegisterClass *RC = TRI.getBoolRC();
277 unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(),
278 RC == &AMDGPU::SReg_64RegClass);
279 I.setDesc(TII.get(InstOpc));
281 // FIXME: Hack to avoid turning the register bank into a register class.
282 // The selector for G_ICMP relies on seeing the register bank for the result
283 // is VCC. In wave32 if we constrain the registers to SReg_32 here, it will
284 // be ambiguous whether it's a scalar or vector bool.
285 if (Src0.isUndef() && !MRI.getRegClassOrNull(Src0.getReg()))
286 MRI.setRegClass(Src0.getReg(), RC);
287 if (Src1.isUndef() && !MRI.getRegClassOrNull(Src1.getReg()))
288 MRI.setRegClass(Src1.getReg(), RC);
290 return RBI.constrainGenericRegister(DstReg, *RC, MRI);
293 // TODO: Should this allow an SCC bank result, and produce a copy from SCC for
295 if (DstRB->getID() == AMDGPU::SGPRRegBankID) {
296 unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(), Size > 32);
297 I.setDesc(TII.get(InstOpc));
299 const TargetRegisterClass *RC
300 = TRI.getConstrainedRegClassForOperand(Dst, MRI);
303 return RBI.constrainGenericRegister(DstReg, *RC, MRI) &&
304 RBI.constrainGenericRegister(Src0.getReg(), *RC, MRI) &&
305 RBI.constrainGenericRegister(Src1.getReg(), *RC, MRI);
311 bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
312 MachineBasicBlock *BB = I.getParent();
313 MachineFunction *MF = BB->getParent();
314 MachineRegisterInfo &MRI = MF->getRegInfo();
315 Register DstReg = I.getOperand(0).getReg();
316 const DebugLoc &DL = I.getDebugLoc();
317 unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
318 const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
319 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
320 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
324 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
326 BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
327 .add(I.getOperand(1))
328 .add(I.getOperand(2));
330 return constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
333 if (STI.hasAddNoCarry()) {
334 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
335 I.setDesc(TII.get(Opc));
336 I.addOperand(*MF, MachineOperand::CreateImm(0));
337 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
338 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
341 const unsigned Opc = Sub ? AMDGPU::V_SUB_I32_e64 : AMDGPU::V_ADD_I32_e64;
343 Register UnusedCarry = MRI.createVirtualRegister(TRI.getWaveMaskRegClass());
345 = BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
346 .addDef(UnusedCarry, RegState::Dead)
347 .add(I.getOperand(1))
348 .add(I.getOperand(2))
351 return constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
354 assert(!Sub && "illegal sub should not reach here");
356 const TargetRegisterClass &RC
357 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
358 const TargetRegisterClass &HalfRC
359 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
361 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0));
362 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0));
363 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1));
364 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1));
366 Register DstLo = MRI.createVirtualRegister(&HalfRC);
367 Register DstHi = MRI.createVirtualRegister(&HalfRC);
370 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo)
373 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi)
377 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
378 Register CarryReg = MRI.createVirtualRegister(CarryRC);
379 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_I32_e64), DstLo)
384 MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi)
385 .addDef(MRI.createVirtualRegister(CarryRC), RegState::Dead)
388 .addReg(CarryReg, RegState::Kill)
391 if (!constrainSelectedInstRegOperands(*Addc, TII, TRI, RBI))
395 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
397 .addImm(AMDGPU::sub0)
399 .addImm(AMDGPU::sub1);
402 if (!RBI.constrainGenericRegister(DstReg, RC, MRI))
409 bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
410 MachineBasicBlock *BB = I.getParent();
411 MachineFunction *MF = BB->getParent();
412 MachineRegisterInfo &MRI = MF->getRegInfo();
413 assert(I.getOperand(2).getImm() % 32 == 0);
414 unsigned SubReg = TRI.getSubRegFromChannel(I.getOperand(2).getImm() / 32);
415 const DebugLoc &DL = I.getDebugLoc();
416 MachineInstr *Copy = BuildMI(*BB, &I, DL, TII.get(TargetOpcode::COPY),
417 I.getOperand(0).getReg())
418 .addReg(I.getOperand(1).getReg(), 0, SubReg);
420 for (const MachineOperand &MO : Copy->operands()) {
421 const TargetRegisterClass *RC =
422 TRI.getConstrainedRegClassForOperand(MO, MRI);
425 RBI.constrainGenericRegister(MO.getReg(), *RC, MRI);
431 bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
432 MachineBasicBlock *BB = MI.getParent();
433 MachineFunction *MF = BB->getParent();
434 MachineRegisterInfo &MRI = MF->getRegInfo();
435 Register DstReg = MI.getOperand(0).getReg();
436 LLT DstTy = MRI.getType(DstReg);
437 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
439 const unsigned SrcSize = SrcTy.getSizeInBits();
443 const DebugLoc &DL = MI.getDebugLoc();
444 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, TRI);
445 const unsigned DstSize = DstTy.getSizeInBits();
446 const TargetRegisterClass *DstRC =
447 TRI.getRegClassForSizeOnBank(DstSize, *DstBank, MRI);
451 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, SrcSize / 8);
452 MachineInstrBuilder MIB =
453 BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
454 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
455 MachineOperand &Src = MI.getOperand(I + 1);
456 MIB.addReg(Src.getReg(), getUndefRegState(Src.isUndef()));
457 MIB.addImm(SubRegs[I]);
459 const TargetRegisterClass *SrcRC
460 = TRI.getConstrainedRegClassForOperand(Src, MRI);
461 if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, MRI))
465 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI))
468 MI.eraseFromParent();
472 bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
473 MachineBasicBlock *BB = MI.getParent();
474 MachineFunction *MF = BB->getParent();
475 MachineRegisterInfo &MRI = MF->getRegInfo();
476 const int NumDst = MI.getNumOperands() - 1;
478 MachineOperand &Src = MI.getOperand(NumDst);
480 Register SrcReg = Src.getReg();
481 Register DstReg0 = MI.getOperand(0).getReg();
482 LLT DstTy = MRI.getType(DstReg0);
483 LLT SrcTy = MRI.getType(SrcReg);
485 const unsigned DstSize = DstTy.getSizeInBits();
486 const unsigned SrcSize = SrcTy.getSizeInBits();
487 const DebugLoc &DL = MI.getDebugLoc();
488 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, TRI);
490 const TargetRegisterClass *SrcRC =
491 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, MRI);
492 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI))
495 const unsigned SrcFlags = getUndefRegState(Src.isUndef());
497 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
498 // source, and this relies on the fact that the same subregister indices are
500 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSize / 8);
501 for (int I = 0, E = NumDst; I != E; ++I) {
502 MachineOperand &Dst = MI.getOperand(I);
503 BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
504 .addReg(SrcReg, SrcFlags, SubRegs[I]);
506 const TargetRegisterClass *DstRC =
507 TRI.getConstrainedRegClassForOperand(Dst, MRI);
508 if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, MRI))
512 MI.eraseFromParent();
516 bool AMDGPUInstructionSelector::selectG_GEP(MachineInstr &I) const {
517 return selectG_ADD_SUB(I);
520 bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
521 MachineBasicBlock *BB = I.getParent();
522 MachineFunction *MF = BB->getParent();
523 MachineRegisterInfo &MRI = MF->getRegInfo();
524 const MachineOperand &MO = I.getOperand(0);
526 // FIXME: Interface for getConstrainedRegClassForOperand needs work. The
527 // regbank check here is to know why getConstrainedRegClassForOperand failed.
528 const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, MRI);
529 if ((!RC && !MRI.getRegBankOrNull(MO.getReg())) ||
530 (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, MRI))) {
531 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
538 bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
539 MachineBasicBlock *BB = I.getParent();
540 MachineFunction *MF = BB->getParent();
541 MachineRegisterInfo &MRI = MF->getRegInfo();
542 unsigned SubReg = TRI.getSubRegFromChannel(I.getOperand(3).getImm() / 32);
543 DebugLoc DL = I.getDebugLoc();
544 MachineInstr *Ins = BuildMI(*BB, &I, DL, TII.get(TargetOpcode::INSERT_SUBREG))
545 .addDef(I.getOperand(0).getReg())
546 .addReg(I.getOperand(1).getReg())
547 .addReg(I.getOperand(2).getReg())
550 for (const MachineOperand &MO : Ins->operands()) {
553 if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
556 const TargetRegisterClass *RC =
557 TRI.getConstrainedRegClassForOperand(MO, MRI);
560 RBI.constrainGenericRegister(MO.getReg(), *RC, MRI);
566 bool AMDGPUInstructionSelector::selectG_INTRINSIC(
567 MachineInstr &I, CodeGenCoverage &CoverageInfo) const {
568 unsigned IntrinsicID = I.getOperand(I.getNumExplicitDefs()).getIntrinsicID();
569 switch (IntrinsicID) {
570 case Intrinsic::maxnum:
571 case Intrinsic::minnum:
572 case Intrinsic::amdgcn_cvt_pkrtz:
573 return selectImpl(I, CoverageInfo);
574 case Intrinsic::amdgcn_if_break: {
575 MachineBasicBlock *BB = I.getParent();
576 MachineFunction *MF = BB->getParent();
577 MachineRegisterInfo &MRI = MF->getRegInfo();
579 // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick
580 // SelectionDAG uses for wave32 vs wave64.
581 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK))
582 .add(I.getOperand(0))
583 .add(I.getOperand(2))
584 .add(I.getOperand(3));
586 Register DstReg = I.getOperand(0).getReg();
587 Register Src0Reg = I.getOperand(2).getReg();
588 Register Src1Reg = I.getOperand(3).getReg();
592 for (Register Reg : { DstReg, Src0Reg, Src1Reg }) {
593 if (!MRI.getRegClassOrNull(Reg))
594 MRI.setRegClass(Reg, TRI.getWaveMaskRegClass());
600 return selectImpl(I, CoverageInfo);
604 static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size) {
605 if (Size != 32 && Size != 64)
609 llvm_unreachable("Unknown condition code!");
610 case CmpInst::ICMP_NE:
611 return Size == 32 ? AMDGPU::V_CMP_NE_U32_e64 : AMDGPU::V_CMP_NE_U64_e64;
612 case CmpInst::ICMP_EQ:
613 return Size == 32 ? AMDGPU::V_CMP_EQ_U32_e64 : AMDGPU::V_CMP_EQ_U64_e64;
614 case CmpInst::ICMP_SGT:
615 return Size == 32 ? AMDGPU::V_CMP_GT_I32_e64 : AMDGPU::V_CMP_GT_I64_e64;
616 case CmpInst::ICMP_SGE:
617 return Size == 32 ? AMDGPU::V_CMP_GE_I32_e64 : AMDGPU::V_CMP_GE_I64_e64;
618 case CmpInst::ICMP_SLT:
619 return Size == 32 ? AMDGPU::V_CMP_LT_I32_e64 : AMDGPU::V_CMP_LT_I64_e64;
620 case CmpInst::ICMP_SLE:
621 return Size == 32 ? AMDGPU::V_CMP_LE_I32_e64 : AMDGPU::V_CMP_LE_I64_e64;
622 case CmpInst::ICMP_UGT:
623 return Size == 32 ? AMDGPU::V_CMP_GT_U32_e64 : AMDGPU::V_CMP_GT_U64_e64;
624 case CmpInst::ICMP_UGE:
625 return Size == 32 ? AMDGPU::V_CMP_GE_U32_e64 : AMDGPU::V_CMP_GE_U64_e64;
626 case CmpInst::ICMP_ULT:
627 return Size == 32 ? AMDGPU::V_CMP_LT_U32_e64 : AMDGPU::V_CMP_LT_U64_e64;
628 case CmpInst::ICMP_ULE:
629 return Size == 32 ? AMDGPU::V_CMP_LE_U32_e64 : AMDGPU::V_CMP_LE_U64_e64;
633 int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
634 unsigned Size) const {
636 if (!STI.hasScalarCompareEq64())
640 case CmpInst::ICMP_NE:
641 return AMDGPU::S_CMP_LG_U64;
642 case CmpInst::ICMP_EQ:
643 return AMDGPU::S_CMP_EQ_U64;
653 case CmpInst::ICMP_NE:
654 return AMDGPU::S_CMP_LG_U32;
655 case CmpInst::ICMP_EQ:
656 return AMDGPU::S_CMP_EQ_U32;
657 case CmpInst::ICMP_SGT:
658 return AMDGPU::S_CMP_GT_I32;
659 case CmpInst::ICMP_SGE:
660 return AMDGPU::S_CMP_GE_I32;
661 case CmpInst::ICMP_SLT:
662 return AMDGPU::S_CMP_LT_I32;
663 case CmpInst::ICMP_SLE:
664 return AMDGPU::S_CMP_LE_I32;
665 case CmpInst::ICMP_UGT:
666 return AMDGPU::S_CMP_GT_U32;
667 case CmpInst::ICMP_UGE:
668 return AMDGPU::S_CMP_GE_U32;
669 case CmpInst::ICMP_ULT:
670 return AMDGPU::S_CMP_LT_U32;
671 case CmpInst::ICMP_ULE:
672 return AMDGPU::S_CMP_LE_U32;
674 llvm_unreachable("Unknown condition code!");
678 bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
679 MachineBasicBlock *BB = I.getParent();
680 MachineFunction *MF = BB->getParent();
681 MachineRegisterInfo &MRI = MF->getRegInfo();
682 const DebugLoc &DL = I.getDebugLoc();
684 unsigned SrcReg = I.getOperand(2).getReg();
685 unsigned Size = RBI.getSizeInBits(SrcReg, MRI, TRI);
687 auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
689 unsigned CCReg = I.getOperand(0).getReg();
690 if (isSCC(CCReg, MRI)) {
691 int Opcode = getS_CMPOpcode(Pred, Size);
694 MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode))
695 .add(I.getOperand(2))
696 .add(I.getOperand(3));
697 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg)
698 .addReg(AMDGPU::SCC);
700 constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI) &&
701 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, MRI);
706 int Opcode = getV_CMPOpcode(Pred, Size);
710 MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode),
711 I.getOperand(0).getReg())
712 .add(I.getOperand(2))
713 .add(I.getOperand(3));
714 RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(),
715 *TRI.getBoolRC(), MRI);
716 bool Ret = constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
721 static MachineInstr *
722 buildEXP(const TargetInstrInfo &TII, MachineInstr *Insert, unsigned Tgt,
723 unsigned Reg0, unsigned Reg1, unsigned Reg2, unsigned Reg3,
724 unsigned VM, bool Compr, unsigned Enabled, bool Done) {
725 const DebugLoc &DL = Insert->getDebugLoc();
726 MachineBasicBlock &BB = *Insert->getParent();
727 unsigned Opcode = Done ? AMDGPU::EXP_DONE : AMDGPU::EXP;
728 return BuildMI(BB, Insert, DL, TII.get(Opcode))
739 bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
740 MachineInstr &I, CodeGenCoverage &CoverageInfo) const {
741 MachineBasicBlock *BB = I.getParent();
742 MachineFunction *MF = BB->getParent();
743 MachineRegisterInfo &MRI = MF->getRegInfo();
745 unsigned IntrinsicID = I.getOperand(0).getIntrinsicID();
746 switch (IntrinsicID) {
747 case Intrinsic::amdgcn_exp: {
748 int64_t Tgt = getConstant(MRI.getVRegDef(I.getOperand(1).getReg()));
749 int64_t Enabled = getConstant(MRI.getVRegDef(I.getOperand(2).getReg()));
750 int64_t Done = getConstant(MRI.getVRegDef(I.getOperand(7).getReg()));
751 int64_t VM = getConstant(MRI.getVRegDef(I.getOperand(8).getReg()));
753 MachineInstr *Exp = buildEXP(TII, &I, Tgt, I.getOperand(3).getReg(),
754 I.getOperand(4).getReg(),
755 I.getOperand(5).getReg(),
756 I.getOperand(6).getReg(),
757 VM, false, Enabled, Done);
760 return constrainSelectedInstRegOperands(*Exp, TII, TRI, RBI);
762 case Intrinsic::amdgcn_exp_compr: {
763 const DebugLoc &DL = I.getDebugLoc();
764 int64_t Tgt = getConstant(MRI.getVRegDef(I.getOperand(1).getReg()));
765 int64_t Enabled = getConstant(MRI.getVRegDef(I.getOperand(2).getReg()));
766 unsigned Reg0 = I.getOperand(3).getReg();
767 unsigned Reg1 = I.getOperand(4).getReg();
768 unsigned Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
769 int64_t Done = getConstant(MRI.getVRegDef(I.getOperand(5).getReg()));
770 int64_t VM = getConstant(MRI.getVRegDef(I.getOperand(6).getReg()));
772 BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef);
773 MachineInstr *Exp = buildEXP(TII, &I, Tgt, Reg0, Reg1, Undef, Undef, VM,
774 true, Enabled, Done);
777 return constrainSelectedInstRegOperands(*Exp, TII, TRI, RBI);
779 case Intrinsic::amdgcn_end_cf: {
780 // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick
781 // SelectionDAG uses for wave32 vs wave64.
782 BuildMI(*BB, &I, I.getDebugLoc(),
783 TII.get(AMDGPU::SI_END_CF))
784 .add(I.getOperand(1));
786 Register Reg = I.getOperand(1).getReg();
789 if (!MRI.getRegClassOrNull(Reg))
790 MRI.setRegClass(Reg, TRI.getWaveMaskRegClass());
794 return selectImpl(I, CoverageInfo);
798 bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
799 MachineBasicBlock *BB = I.getParent();
800 MachineFunction *MF = BB->getParent();
801 MachineRegisterInfo &MRI = MF->getRegInfo();
802 const DebugLoc &DL = I.getDebugLoc();
804 unsigned DstReg = I.getOperand(0).getReg();
805 unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
806 assert(Size <= 32 || Size == 64);
807 const MachineOperand &CCOp = I.getOperand(1);
808 unsigned CCReg = CCOp.getReg();
809 if (isSCC(CCReg, MRI)) {
810 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
811 AMDGPU::S_CSELECT_B32;
812 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
815 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
816 // bank, because it does not cover the register class that we used to represent
817 // for it. So we need to manually set the register class here.
818 if (!MRI.getRegClassOrNull(CCReg))
819 MRI.setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, MRI));
820 MachineInstr *Select = BuildMI(*BB, &I, DL, TII.get(SelectOpcode), DstReg)
821 .add(I.getOperand(2))
822 .add(I.getOperand(3));
824 bool Ret = constrainSelectedInstRegOperands(*Select, TII, TRI, RBI) |
825 constrainSelectedInstRegOperands(*CopySCC, TII, TRI, RBI);
830 // Wide VGPR select should have been split in RegBankSelect.
834 MachineInstr *Select =
835 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
837 .add(I.getOperand(3))
839 .add(I.getOperand(2))
840 .add(I.getOperand(1));
842 bool Ret = constrainSelectedInstRegOperands(*Select, TII, TRI, RBI);
847 bool AMDGPUInstructionSelector::selectG_STORE(MachineInstr &I) const {
848 MachineBasicBlock *BB = I.getParent();
849 MachineFunction *MF = BB->getParent();
850 MachineRegisterInfo &MRI = MF->getRegInfo();
851 DebugLoc DL = I.getDebugLoc();
852 unsigned PtrSize = RBI.getSizeInBits(I.getOperand(1).getReg(), MRI, TRI);
854 LLVM_DEBUG(dbgs() << "Unhandled address space\n");
858 unsigned StoreSize = RBI.getSizeInBits(I.getOperand(0).getReg(), MRI, TRI);
861 // FIXME: Remove this when integers > s32 naturally selected.
866 Opcode = AMDGPU::FLAT_STORE_DWORD;
869 Opcode = AMDGPU::FLAT_STORE_DWORDX2;
872 Opcode = AMDGPU::FLAT_STORE_DWORDX3;
875 Opcode = AMDGPU::FLAT_STORE_DWORDX4;
879 MachineInstr *Flat = BuildMI(*BB, &I, DL, TII.get(Opcode))
880 .add(I.getOperand(1))
881 .add(I.getOperand(0))
888 // Now that we selected an opcode, we need to constrain the register
889 // operands to use appropriate classes.
890 bool Ret = constrainSelectedInstRegOperands(*Flat, TII, TRI, RBI);
896 static int sizeToSubRegIndex(unsigned Size) {
901 return AMDGPU::sub0_sub1;
903 return AMDGPU::sub0_sub1_sub2;
905 return AMDGPU::sub0_sub1_sub2_sub3;
907 return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7;
913 return sizeToSubRegIndex(PowerOf2Ceil(Size));
917 bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
918 MachineBasicBlock *BB = I.getParent();
919 MachineFunction *MF = BB->getParent();
920 MachineRegisterInfo &MRI = MF->getRegInfo();
922 unsigned DstReg = I.getOperand(0).getReg();
923 unsigned SrcReg = I.getOperand(1).getReg();
924 const LLT DstTy = MRI.getType(DstReg);
925 const LLT SrcTy = MRI.getType(SrcReg);
926 if (!DstTy.isScalar())
929 const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
930 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, MRI, TRI);
934 unsigned DstSize = DstTy.getSizeInBits();
935 unsigned SrcSize = SrcTy.getSizeInBits();
937 const TargetRegisterClass *SrcRC
938 = TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB, MRI);
939 const TargetRegisterClass *DstRC
940 = TRI.getRegClassForSizeOnBank(DstSize, *DstRB, MRI);
943 int SubRegIdx = sizeToSubRegIndex(DstSize);
947 // Deal with weird cases where the class only partially supports the subreg
949 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
953 I.getOperand(1).setSubReg(SubRegIdx);
956 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
957 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
958 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
962 I.setDesc(TII.get(TargetOpcode::COPY));
966 /// \returns true if a bitmask for \p Size bits will be an inline immediate.
967 static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
968 Mask = maskTrailingOnes<unsigned>(Size);
969 int SignedMask = static_cast<int>(Mask);
970 return SignedMask >= -16 && SignedMask <= 64;
973 bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
974 bool Signed = I.getOpcode() == AMDGPU::G_SEXT;
975 const DebugLoc &DL = I.getDebugLoc();
976 MachineBasicBlock &MBB = *I.getParent();
977 MachineFunction &MF = *MBB.getParent();
978 MachineRegisterInfo &MRI = MF.getRegInfo();
979 const unsigned DstReg = I.getOperand(0).getReg();
980 const unsigned SrcReg = I.getOperand(1).getReg();
982 const LLT DstTy = MRI.getType(DstReg);
983 const LLT SrcTy = MRI.getType(SrcReg);
984 const LLT S1 = LLT::scalar(1);
985 const unsigned SrcSize = SrcTy.getSizeInBits();
986 const unsigned DstSize = DstTy.getSizeInBits();
987 if (!DstTy.isScalar())
990 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, TRI);
992 if (SrcBank->getID() == AMDGPU::SCCRegBankID) {
993 if (SrcTy != S1 || DstSize > 64) // Invalid
997 DstSize > 32 ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
998 const TargetRegisterClass *DstRC =
999 DstSize > 32 ? &AMDGPU::SReg_64RegClass : &AMDGPU::SReg_32RegClass;
1001 // FIXME: Create an extra copy to avoid incorrectly constraining the result
1002 // of the scc producer.
1003 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
1004 BuildMI(MBB, I, DL, TII.get(AMDGPU::COPY), TmpReg)
1006 BuildMI(MBB, I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
1009 // The instruction operands are backwards from what you would expect.
1010 BuildMI(MBB, I, DL, TII.get(Opcode), DstReg)
1012 .addImm(Signed ? -1 : 1);
1013 return RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
1016 if (SrcBank->getID() == AMDGPU::VCCRegBankID && DstSize <= 32) {
1017 if (SrcTy != S1) // Invalid
1020 MachineInstr *ExtI =
1021 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1022 .addImm(0) // src0_modifiers
1024 .addImm(0) // src1_modifiers
1025 .addImm(Signed ? -1 : 1) // src1
1027 return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
1030 if (I.getOpcode() == AMDGPU::G_ANYEXT)
1031 return selectCOPY(I);
1033 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
1034 // 64-bit should have been split up in RegBankSelect
1036 // Try to use an and with a mask if it will save code size.
1038 if (!Signed && shouldUseAndMask(SrcSize, Mask)) {
1039 MachineInstr *ExtI =
1040 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg)
1043 return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
1046 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32 : AMDGPU::V_BFE_U32;
1047 MachineInstr *ExtI =
1048 BuildMI(MBB, I, DL, TII.get(BFE), DstReg)
1050 .addImm(0) // Offset
1051 .addImm(SrcSize); // Width
1052 return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
1055 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
1056 if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI))
1059 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
1060 const unsigned SextOpc = SrcSize == 8 ?
1061 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
1062 BuildMI(MBB, I, DL, TII.get(SextOpc), DstReg)
1064 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, MRI);
1067 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
1068 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1070 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
1071 if (DstSize > 32 && SrcSize <= 32) {
1072 // We need a 64-bit register source, but the high bits don't matter.
1074 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1076 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
1077 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
1078 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg)
1080 .addImm(AMDGPU::sub0)
1082 .addImm(AMDGPU::sub1);
1084 BuildMI(MBB, I, DL, TII.get(BFE64), DstReg)
1086 .addImm(SrcSize << 16);
1088 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, MRI);
1092 if (!Signed && shouldUseAndMask(SrcSize, Mask)) {
1093 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg)
1097 BuildMI(MBB, I, DL, TII.get(BFE32), DstReg)
1099 .addImm(SrcSize << 16);
1102 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, MRI);
1108 bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
1109 MachineBasicBlock *BB = I.getParent();
1110 MachineFunction *MF = BB->getParent();
1111 MachineRegisterInfo &MRI = MF->getRegInfo();
1112 MachineOperand &ImmOp = I.getOperand(1);
1114 // The AMDGPU backend only supports Imm operands and not CImm or FPImm.
1115 if (ImmOp.isFPImm()) {
1116 const APInt &Imm = ImmOp.getFPImm()->getValueAPF().bitcastToAPInt();
1117 ImmOp.ChangeToImmediate(Imm.getZExtValue());
1118 } else if (ImmOp.isCImm()) {
1119 ImmOp.ChangeToImmediate(ImmOp.getCImm()->getZExtValue());
1122 unsigned DstReg = I.getOperand(0).getReg();
1125 const RegisterBank *RB = MRI.getRegBankOrNull(I.getOperand(0).getReg());
1127 IsSgpr = RB->getID() == AMDGPU::SGPRRegBankID;
1128 Size = MRI.getType(DstReg).getSizeInBits();
1130 const TargetRegisterClass *RC = TRI.getRegClassForReg(MRI, DstReg);
1131 IsSgpr = TRI.isSGPRClass(RC);
1132 Size = TRI.getRegSizeInBits(*RC);
1135 if (Size != 32 && Size != 64)
1138 unsigned Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1140 I.setDesc(TII.get(Opcode));
1141 I.addImplicitDefUseOperands(*MF);
1142 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1145 DebugLoc DL = I.getDebugLoc();
1146 const TargetRegisterClass *RC = IsSgpr ? &AMDGPU::SReg_32_XM0RegClass :
1147 &AMDGPU::VGPR_32RegClass;
1148 unsigned LoReg = MRI.createVirtualRegister(RC);
1149 unsigned HiReg = MRI.createVirtualRegister(RC);
1150 const APInt &Imm = APInt(Size, I.getOperand(1).getImm());
1152 BuildMI(*BB, &I, DL, TII.get(Opcode), LoReg)
1153 .addImm(Imm.trunc(32).getZExtValue());
1155 BuildMI(*BB, &I, DL, TII.get(Opcode), HiReg)
1156 .addImm(Imm.ashr(32).getZExtValue());
1158 const MachineInstr *RS =
1159 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
1161 .addImm(AMDGPU::sub0)
1163 .addImm(AMDGPU::sub1);
1165 // We can't call constrainSelectedInstRegOperands here, because it doesn't
1166 // work for target independent opcodes
1167 I.eraseFromParent();
1168 const TargetRegisterClass *DstRC =
1169 TRI.getConstrainedRegClassForOperand(RS->getOperand(0), MRI);
1172 return RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
1175 static bool isConstant(const MachineInstr &MI) {
1176 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
1179 void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
1180 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
1182 const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg());
1186 if (PtrMI->getOpcode() != TargetOpcode::G_GEP)
1189 GEPInfo GEPInfo(*PtrMI);
1191 for (unsigned i = 1, e = 3; i < e; ++i) {
1192 const MachineOperand &GEPOp = PtrMI->getOperand(i);
1193 const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg());
1195 if (isConstant(*OpDef)) {
1196 // FIXME: Is it possible to have multiple Imm parts? Maybe if we
1197 // are lacking other optimizations.
1198 assert(GEPInfo.Imm == 0);
1199 GEPInfo.Imm = OpDef->getOperand(1).getCImm()->getSExtValue();
1202 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI);
1203 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
1204 GEPInfo.SgprParts.push_back(GEPOp.getReg());
1206 GEPInfo.VgprParts.push_back(GEPOp.getReg());
1209 AddrInfo.push_back(GEPInfo);
1210 getAddrModeInfo(*PtrMI, MRI, AddrInfo);
1213 bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
1214 if (!MI.hasOneMemOperand())
1217 const MachineMemOperand *MMO = *MI.memoperands_begin();
1218 const Value *Ptr = MMO->getValue();
1220 // UndefValue means this is a load of a kernel input. These are uniform.
1221 // Sometimes LDS instructions have constant pointers.
1222 // If Ptr is null, then that means this mem operand contains a
1223 // PseudoSourceValue like GOT.
1224 if (!Ptr || isa<UndefValue>(Ptr) || isa<Argument>(Ptr) ||
1225 isa<Constant>(Ptr) || isa<GlobalValue>(Ptr))
1228 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
1231 const Instruction *I = dyn_cast<Instruction>(Ptr);
1232 return I && I->getMetadata("amdgpu.uniform");
1235 bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
1236 for (const GEPInfo &GEPInfo : AddrInfo) {
1237 if (!GEPInfo.VgprParts.empty())
1243 bool AMDGPUInstructionSelector::selectG_LOAD(MachineInstr &I) const {
1244 // TODO: Can/should we insert m0 initialization here for DS instructions and
1245 // call the normal selector?
1249 bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
1250 MachineBasicBlock *BB = I.getParent();
1251 MachineFunction *MF = BB->getParent();
1252 MachineRegisterInfo &MRI = MF->getRegInfo();
1253 MachineOperand &CondOp = I.getOperand(0);
1254 Register CondReg = CondOp.getReg();
1255 const DebugLoc &DL = I.getDebugLoc();
1258 Register CondPhysReg;
1259 const TargetRegisterClass *ConstrainRC;
1261 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
1262 // whether the branch is uniform when selecting the instruction. In
1263 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
1264 // RegBankSelect knows what it's doing if the branch condition is scc, even
1265 // though it currently does not.
1266 if (isSCC(CondReg, MRI)) {
1267 CondPhysReg = AMDGPU::SCC;
1268 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
1269 ConstrainRC = &AMDGPU::SReg_32_XM0RegClass;
1270 } else if (isVCC(CondReg, MRI)) {
1271 // FIXME: Do we have to insert an and with exec here, like in SelectionDAG?
1272 // We sort of know that a VCC producer based on the register bank, that ands
1273 // inactive lanes with 0. What if there was a logical operation with vcc
1274 // producers in different blocks/with different exec masks?
1275 // FIXME: Should scc->vcc copies and with exec?
1276 CondPhysReg = TRI.getVCC();
1277 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
1278 ConstrainRC = TRI.getBoolRC();
1282 if (!MRI.getRegClassOrNull(CondReg))
1283 MRI.setRegClass(CondReg, ConstrainRC);
1285 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg)
1287 BuildMI(*BB, &I, DL, TII.get(BrOpcode))
1288 .addMBB(I.getOperand(1).getMBB());
1290 I.eraseFromParent();
1294 bool AMDGPUInstructionSelector::selectG_FRAME_INDEX(MachineInstr &I) const {
1295 MachineBasicBlock *BB = I.getParent();
1296 MachineFunction *MF = BB->getParent();
1297 MachineRegisterInfo &MRI = MF->getRegInfo();
1299 Register DstReg = I.getOperand(0).getReg();
1300 const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
1301 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
1302 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
1304 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
1306 return RBI.constrainGenericRegister(
1307 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI);
1310 bool AMDGPUInstructionSelector::select(MachineInstr &I,
1311 CodeGenCoverage &CoverageInfo) const {
1313 return selectPHI(I);
1315 if (!isPreISelGenericOpcode(I.getOpcode())) {
1317 return selectCOPY(I);
1321 switch (I.getOpcode()) {
1322 case TargetOpcode::G_AND:
1323 case TargetOpcode::G_OR:
1324 case TargetOpcode::G_XOR:
1325 if (selectG_AND_OR_XOR(I))
1327 return selectImpl(I, CoverageInfo);
1328 case TargetOpcode::G_ADD:
1329 case TargetOpcode::G_SUB:
1330 if (selectG_ADD_SUB(I))
1334 return selectImpl(I, CoverageInfo);
1335 case TargetOpcode::G_INTTOPTR:
1336 case TargetOpcode::G_BITCAST:
1337 return selectCOPY(I);
1338 case TargetOpcode::G_CONSTANT:
1339 case TargetOpcode::G_FCONSTANT:
1340 return selectG_CONSTANT(I);
1341 case TargetOpcode::G_EXTRACT:
1342 return selectG_EXTRACT(I);
1343 case TargetOpcode::G_MERGE_VALUES:
1344 case TargetOpcode::G_BUILD_VECTOR:
1345 case TargetOpcode::G_CONCAT_VECTORS:
1346 return selectG_MERGE_VALUES(I);
1347 case TargetOpcode::G_UNMERGE_VALUES:
1348 return selectG_UNMERGE_VALUES(I);
1349 case TargetOpcode::G_GEP:
1350 return selectG_GEP(I);
1351 case TargetOpcode::G_IMPLICIT_DEF:
1352 return selectG_IMPLICIT_DEF(I);
1353 case TargetOpcode::G_INSERT:
1354 return selectG_INSERT(I);
1355 case TargetOpcode::G_INTRINSIC:
1356 return selectG_INTRINSIC(I, CoverageInfo);
1357 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1358 return selectG_INTRINSIC_W_SIDE_EFFECTS(I, CoverageInfo);
1359 case TargetOpcode::G_ICMP:
1360 if (selectG_ICMP(I))
1362 return selectImpl(I, CoverageInfo);
1363 case TargetOpcode::G_LOAD:
1364 return selectImpl(I, CoverageInfo);
1365 case TargetOpcode::G_SELECT:
1366 return selectG_SELECT(I);
1367 case TargetOpcode::G_STORE:
1368 if (selectImpl(I, CoverageInfo))
1370 return selectG_STORE(I);
1371 case TargetOpcode::G_TRUNC:
1372 return selectG_TRUNC(I);
1373 case TargetOpcode::G_SEXT:
1374 case TargetOpcode::G_ZEXT:
1375 case TargetOpcode::G_ANYEXT:
1376 if (selectG_SZA_EXT(I)) {
1377 I.eraseFromParent();
1382 case TargetOpcode::G_BRCOND:
1383 return selectG_BRCOND(I);
1384 case TargetOpcode::G_FRAME_INDEX:
1385 return selectG_FRAME_INDEX(I);
1386 case TargetOpcode::G_FENCE:
1387 // FIXME: Tablegen importer doesn't handle the imm operands correctly, and
1388 // is checking for G_CONSTANT
1389 I.setDesc(TII.get(AMDGPU::ATOMIC_FENCE));
1395 InstructionSelector::ComplexRendererFns
1396 AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
1398 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }
1403 std::pair<Register, unsigned>
1404 AMDGPUInstructionSelector::selectVOP3ModsImpl(
1405 Register Src, const MachineRegisterInfo &MRI) const {
1407 MachineInstr *MI = MRI.getVRegDef(Src);
1409 if (MI && MI->getOpcode() == AMDGPU::G_FNEG) {
1410 Src = MI->getOperand(1).getReg();
1411 Mods |= SISrcMods::NEG;
1412 MI = MRI.getVRegDef(Src);
1415 if (MI && MI->getOpcode() == AMDGPU::G_FABS) {
1416 Src = MI->getOperand(1).getReg();
1417 Mods |= SISrcMods::ABS;
1420 return std::make_pair(Src, Mods);
1424 /// This will select either an SGPR or VGPR operand and will save us from
1425 /// having to write an extra tablegen pattern.
1426 InstructionSelector::ComplexRendererFns
1427 AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
1429 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }
1433 InstructionSelector::ComplexRendererFns
1434 AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
1435 MachineRegisterInfo &MRI
1436 = Root.getParent()->getParent()->getParent()->getRegInfo();
1440 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(), MRI);
1443 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
1444 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
1445 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
1446 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
1449 InstructionSelector::ComplexRendererFns
1450 AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
1452 [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
1453 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
1454 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
1458 InstructionSelector::ComplexRendererFns
1459 AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
1460 MachineRegisterInfo &MRI
1461 = Root.getParent()->getParent()->getParent()->getRegInfo();
1465 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(), MRI);
1468 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
1469 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
1473 InstructionSelector::ComplexRendererFns
1474 AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
1475 MachineRegisterInfo &MRI =
1476 Root.getParent()->getParent()->getParent()->getRegInfo();
1478 SmallVector<GEPInfo, 4> AddrInfo;
1479 getAddrModeInfo(*Root.getParent(), MRI, AddrInfo);
1481 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
1484 const GEPInfo &GEPInfo = AddrInfo[0];
1486 if (!AMDGPU::isLegalSMRDImmOffset(STI, GEPInfo.Imm))
1489 unsigned PtrReg = GEPInfo.SgprParts[0];
1490 int64_t EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPInfo.Imm);
1492 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); },
1493 [=](MachineInstrBuilder &MIB) { MIB.addImm(EncodedImm); }
1497 InstructionSelector::ComplexRendererFns
1498 AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
1499 MachineRegisterInfo &MRI =
1500 Root.getParent()->getParent()->getParent()->getRegInfo();
1502 SmallVector<GEPInfo, 4> AddrInfo;
1503 getAddrModeInfo(*Root.getParent(), MRI, AddrInfo);
1505 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
1508 const GEPInfo &GEPInfo = AddrInfo[0];
1509 unsigned PtrReg = GEPInfo.SgprParts[0];
1510 int64_t EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPInfo.Imm);
1511 if (!isUInt<32>(EncodedImm))
1515 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); },
1516 [=](MachineInstrBuilder &MIB) { MIB.addImm(EncodedImm); }
1520 InstructionSelector::ComplexRendererFns
1521 AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
1522 MachineInstr *MI = Root.getParent();
1523 MachineBasicBlock *MBB = MI->getParent();
1524 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1526 SmallVector<GEPInfo, 4> AddrInfo;
1527 getAddrModeInfo(*MI, MRI, AddrInfo);
1529 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
1530 // then we can select all ptr + 32-bit offsets not just immediate offsets.
1531 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
1534 const GEPInfo &GEPInfo = AddrInfo[0];
1535 if (!GEPInfo.Imm || !isUInt<32>(GEPInfo.Imm))
1538 // If we make it this far we have a load with an 32-bit immediate offset.
1539 // It is OK to select this using a sgpr offset, because we have already
1540 // failed trying to select this load into one of the _IMM variants since
1541 // the _IMM Patterns are considered before the _SGPR patterns.
1542 unsigned PtrReg = GEPInfo.SgprParts[0];
1543 unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
1544 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), OffsetReg)
1545 .addImm(GEPInfo.Imm);
1547 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); },
1548 [=](MachineInstrBuilder &MIB) { MIB.addReg(OffsetReg); }
1552 template <bool Signed>
1553 InstructionSelector::ComplexRendererFns
1554 AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root) const {
1555 MachineInstr *MI = Root.getParent();
1556 MachineBasicBlock *MBB = MI->getParent();
1557 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1559 InstructionSelector::ComplexRendererFns Default = {{
1560 [=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
1561 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // offset
1562 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // slc
1565 if (!STI.hasFlatInstOffsets())
1568 const MachineInstr *OpDef = MRI.getVRegDef(Root.getReg());
1569 if (!OpDef || OpDef->getOpcode() != AMDGPU::G_GEP)
1572 Optional<int64_t> Offset =
1573 getConstantVRegVal(OpDef->getOperand(2).getReg(), MRI);
1574 if (!Offset.hasValue())
1577 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
1578 if (!TII.isLegalFLATOffset(Offset.getValue(), AddrSpace, Signed))
1581 Register BasePtr = OpDef->getOperand(1).getReg();
1584 [=](MachineInstrBuilder &MIB) { MIB.addReg(BasePtr); },
1585 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset.getValue()); },
1586 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // slc
1590 InstructionSelector::ComplexRendererFns
1591 AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
1592 return selectFlatOffsetImpl<false>(Root);
1595 InstructionSelector::ComplexRendererFns
1596 AMDGPUInstructionSelector::selectFlatOffsetSigned(MachineOperand &Root) const {
1597 return selectFlatOffsetImpl<true>(Root);
1601 static bool signBitIsZero(const MachineOperand &Op,
1602 const MachineRegisterInfo &MRI) {
1606 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
1607 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
1608 return PSV && PSV->isStack();
1611 InstructionSelector::ComplexRendererFns
1612 AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
1613 MachineInstr *MI = Root.getParent();
1614 MachineBasicBlock *MBB = MI->getParent();
1615 MachineFunction *MF = MBB->getParent();
1616 MachineRegisterInfo &MRI = MF->getRegInfo();
1617 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1620 if (mi_match(Root.getReg(), MRI, m_ICst(Offset))) {
1621 Register HighBits = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1623 // TODO: Should this be inside the render function? The iterator seems to
1625 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
1627 .addImm(Offset & ~4095);
1629 return {{[=](MachineInstrBuilder &MIB) { // rsrc
1630 MIB.addReg(Info->getScratchRSrcReg());
1632 [=](MachineInstrBuilder &MIB) { // vaddr
1633 MIB.addReg(HighBits);
1635 [=](MachineInstrBuilder &MIB) { // soffset
1636 const MachineMemOperand *MMO = *MI->memoperands_begin();
1637 const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
1639 Register SOffsetReg = isStackPtrRelative(PtrInfo)
1640 ? Info->getStackPtrOffsetReg()
1641 : Info->getScratchWaveOffsetReg();
1642 MIB.addReg(SOffsetReg);
1644 [=](MachineInstrBuilder &MIB) { // offset
1645 MIB.addImm(Offset & 4095);
1649 assert(Offset == 0);
1651 // Try to fold a frame index directly into the MUBUF vaddr field, and any
1654 Register VAddr = Root.getReg();
1655 if (const MachineInstr *RootDef = MRI.getVRegDef(Root.getReg())) {
1656 if (isBaseWithConstantOffset(Root, MRI)) {
1657 const MachineOperand &LHS = RootDef->getOperand(1);
1658 const MachineOperand &RHS = RootDef->getOperand(2);
1659 const MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
1660 const MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
1661 if (LHSDef && RHSDef) {
1662 int64_t PossibleOffset =
1663 RHSDef->getOperand(1).getCImm()->getSExtValue();
1664 if (SIInstrInfo::isLegalMUBUFImmOffset(PossibleOffset) &&
1665 (!STI.privateMemoryResourceIsRangeChecked() ||
1666 signBitIsZero(LHS, MRI))) {
1667 if (LHSDef->getOpcode() == AMDGPU::G_FRAME_INDEX)
1668 FI = LHSDef->getOperand(1).getIndex();
1670 VAddr = LHS.getReg();
1671 Offset = PossibleOffset;
1674 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) {
1675 FI = RootDef->getOperand(1).getIndex();
1679 // If we don't know this private access is a local stack object, it needs to
1680 // be relative to the entry point's scratch wave offset register.
1681 // TODO: Should split large offsets that don't fit like above.
1682 // TODO: Don't use scratch wave offset just because the offset didn't fit.
1683 Register SOffset = FI.hasValue() ? Info->getStackPtrOffsetReg()
1684 : Info->getScratchWaveOffsetReg();
1686 return {{[=](MachineInstrBuilder &MIB) { // rsrc
1687 MIB.addReg(Info->getScratchRSrcReg());
1689 [=](MachineInstrBuilder &MIB) { // vaddr
1691 MIB.addFrameIndex(FI.getValue());
1695 [=](MachineInstrBuilder &MIB) { // soffset
1696 MIB.addReg(SOffset);
1698 [=](MachineInstrBuilder &MIB) { // offset
1703 InstructionSelector::ComplexRendererFns
1704 AMDGPUInstructionSelector::selectMUBUFScratchOffset(
1705 MachineOperand &Root) const {
1706 MachineInstr *MI = Root.getParent();
1707 MachineBasicBlock *MBB = MI->getParent();
1708 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1711 if (!mi_match(Root.getReg(), MRI, m_ICst(Offset)) ||
1712 !SIInstrInfo::isLegalMUBUFImmOffset(Offset))
1715 const MachineFunction *MF = MBB->getParent();
1716 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1717 const MachineMemOperand *MMO = *MI->memoperands_begin();
1718 const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
1720 Register SOffsetReg = isStackPtrRelative(PtrInfo)
1721 ? Info->getStackPtrOffsetReg()
1722 : Info->getScratchWaveOffsetReg();
1724 [=](MachineInstrBuilder &MIB) {
1725 MIB.addReg(Info->getScratchRSrcReg());
1727 [=](MachineInstrBuilder &MIB) { MIB.addReg(SOffsetReg); }, // soffset
1728 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset