1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //==-----------------------------------------------------------------------===//
10 /// Defines an instruction selector for the AMDGPU target.
12 //===----------------------------------------------------------------------===//
14 #include "AMDGPUISelDAGToDAG.h"
16 #include "AMDGPUInstrInfo.h"
17 #include "AMDGPUSubtarget.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "MCTargetDesc/R600MCTargetDesc.h"
21 #include "R600RegisterInfo.h"
22 #include "SIMachineFunctionInfo.h"
23 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
24 #include "llvm/Analysis/ValueTracking.h"
25 #include "llvm/CodeGen/FunctionLoweringInfo.h"
26 #include "llvm/CodeGen/SelectionDAG.h"
27 #include "llvm/CodeGen/SelectionDAGISel.h"
28 #include "llvm/CodeGen/SelectionDAGNodes.h"
29 #include "llvm/IR/IntrinsicsAMDGPU.h"
30 #include "llvm/InitializePasses.h"
32 #ifdef EXPENSIVE_CHECKS
33 #include "llvm/Analysis/LoopInfo.h"
34 #include "llvm/IR/Dominators.h"
37 #define DEBUG_TYPE "amdgpu-isel"
41 //===----------------------------------------------------------------------===//
42 // Instruction Selector Implementation
43 //===----------------------------------------------------------------------===//
46 static SDValue stripBitcast(SDValue Val) {
47 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
50 // Figure out if this is really an extract of the high 16-bits of a dword.
51 static bool isExtractHiElt(SDValue In, SDValue &Out) {
52 In = stripBitcast(In);
54 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
55 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
58 Out = In.getOperand(0);
63 if (In.getOpcode() != ISD::TRUNCATE)
66 SDValue Srl = In.getOperand(0);
67 if (Srl.getOpcode() == ISD::SRL) {
68 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
69 if (ShiftAmt->getZExtValue() == 16) {
70 Out = stripBitcast(Srl.getOperand(0));
79 // Look through operations that obscure just looking at the low 16-bits of the
81 static SDValue stripExtractLoElt(SDValue In) {
82 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
83 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
84 if (Idx->isZero() && In.getValueSizeInBits() <= 32)
85 return In.getOperand(0);
89 if (In.getOpcode() == ISD::TRUNCATE) {
90 SDValue Src = In.getOperand(0);
91 if (Src.getValueType().getSizeInBits() == 32)
92 return stripBitcast(Src);
98 } // end anonymous namespace
100 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
101 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
102 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
103 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
104 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
105 #ifdef EXPENSIVE_CHECKS
106 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
107 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
109 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
110 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
112 /// This pass converts a legalized DAG into a AMDGPU-specific
113 // DAG, ready for instruction scheduling.
114 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM,
115 CodeGenOpt::Level OptLevel) {
116 return new AMDGPUDAGToDAGISel(TM, OptLevel);
119 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM,
120 CodeGenOpt::Level OptLevel)
121 : SelectionDAGISel(ID, TM, OptLevel) {
122 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
125 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
126 #ifdef EXPENSIVE_CHECKS
127 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
128 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
129 for (auto &L : LI->getLoopsInPreorder()) {
130 assert(L->isLCSSAForm(DT));
133 Subtarget = &MF.getSubtarget<GCNSubtarget>();
134 Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction());
135 return SelectionDAGISel::runOnMachineFunction(MF);
138 bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(unsigned Opc) const {
139 // XXX - only need to list legal operations.
146 case ISD::FCANONICALIZE:
147 case ISD::UINT_TO_FP:
148 case ISD::SINT_TO_FP:
150 // Fabs is lowered to a bit operation, but it's an and which will clear the
165 case ISD::FNEARBYINT:
170 case AMDGPUISD::FRACT:
171 case AMDGPUISD::CLAMP:
172 case AMDGPUISD::COS_HW:
173 case AMDGPUISD::SIN_HW:
174 case AMDGPUISD::FMIN3:
175 case AMDGPUISD::FMAX3:
176 case AMDGPUISD::FMED3:
177 case AMDGPUISD::FMAD_FTZ:
180 case AMDGPUISD::RCP_IFLAG:
181 case AMDGPUISD::LDEXP:
182 // On gfx10, all 16-bit instructions preserve the high bits.
183 return Subtarget->getGeneration() <= AMDGPUSubtarget::GFX9;
185 // We may select fptrunc (fma/mad) to mad_mixlo, which does not zero the
186 // high bits on gfx9.
187 // TODO: If we had the source node we could see if the source was fma/mad
188 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
191 case AMDGPUISD::DIV_FIXUP:
192 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
194 // fcopysign, select and others may be lowered to 32-bit bit operations
195 // which don't zero the high bits.
200 void AMDGPUDAGToDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
201 AU.addRequired<AMDGPUArgumentUsageInfo>();
202 AU.addRequired<LegacyDivergenceAnalysis>();
203 #ifdef EXPENSIVE_CHECKS
204 AU.addRequired<DominatorTreeWrapperPass>();
205 AU.addRequired<LoopInfoWrapperPass>();
207 SelectionDAGISel::getAnalysisUsage(AU);
210 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
211 assert(Subtarget->d16PreservesUnusedBits());
212 MVT VT = N->getValueType(0).getSimpleVT();
213 if (VT != MVT::v2i16 && VT != MVT::v2f16)
216 SDValue Lo = N->getOperand(0);
217 SDValue Hi = N->getOperand(1);
219 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
221 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
222 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
223 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
225 // Need to check for possible indirect dependencies on the other half of the
226 // vector to avoid introducing a cycle.
227 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
228 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
230 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
232 LdHi->getChain(), LdHi->getBasePtr(), TiedIn
235 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
236 if (LdHi->getMemoryVT() == MVT::i8) {
237 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
238 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
240 assert(LdHi->getMemoryVT() == MVT::i16);
244 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
245 Ops, LdHi->getMemoryVT(),
246 LdHi->getMemOperand());
248 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
249 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
253 // build_vector (load ptr), hi -> load_d16_lo ptr, hi
254 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
255 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
256 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
257 if (LdLo && Lo.hasOneUse()) {
258 SDValue TiedIn = getHi16Elt(Hi);
259 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
262 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
263 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
264 if (LdLo->getMemoryVT() == MVT::i8) {
265 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
266 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
268 assert(LdLo->getMemoryVT() == MVT::i16);
271 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
274 LdLo->getChain(), LdLo->getBasePtr(), TiedIn
278 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
279 Ops, LdLo->getMemoryVT(),
280 LdLo->getMemOperand());
282 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
283 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
290 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
291 if (!Subtarget->d16PreservesUnusedBits())
294 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
296 bool MadeChange = false;
297 while (Position != CurDAG->allnodes_begin()) {
298 SDNode *N = &*--Position;
302 switch (N->getOpcode()) {
303 case ISD::BUILD_VECTOR:
304 MadeChange |= matchLoadD16FromBuildVector(N);
312 CurDAG->RemoveDeadNodes();
313 LLVM_DEBUG(dbgs() << "After PreProcess:\n";
318 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
319 bool Negated) const {
323 const SIInstrInfo *TII = Subtarget->getInstrInfo();
325 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
326 return TII->isInlineConstant(-C->getAPIntValue());
328 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
329 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
332 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
333 return TII->isInlineConstant(C->getAPIntValue());
335 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
336 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
342 /// Determine the register class for \p OpNo
343 /// \returns The register class of the virtual register that will be used for
344 /// the given operand number \OpNo or NULL if the register class cannot be
346 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
347 unsigned OpNo) const {
348 if (!N->isMachineOpcode()) {
349 if (N->getOpcode() == ISD::CopyToReg) {
350 Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
351 if (Reg.isVirtual()) {
352 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
353 return MRI.getRegClass(Reg);
356 const SIRegisterInfo *TRI
357 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
358 return TRI->getPhysRegBaseClass(Reg);
364 switch (N->getMachineOpcode()) {
366 const MCInstrDesc &Desc =
367 Subtarget->getInstrInfo()->get(N->getMachineOpcode());
368 unsigned OpIdx = Desc.getNumDefs() + OpNo;
369 if (OpIdx >= Desc.getNumOperands())
371 int RegClass = Desc.operands()[OpIdx].RegClass;
375 return Subtarget->getRegisterInfo()->getRegClass(RegClass);
377 case AMDGPU::REG_SEQUENCE: {
378 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
379 const TargetRegisterClass *SuperRC =
380 Subtarget->getRegisterInfo()->getRegClass(RCID);
382 SDValue SubRegOp = N->getOperand(OpNo + 1);
383 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
384 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
390 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
391 SDValue Glue) const {
392 SmallVector <SDValue, 8> Ops;
393 Ops.push_back(NewChain); // Replace the chain.
394 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
395 Ops.push_back(N->getOperand(i));
398 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
401 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
402 const SITargetLowering& Lowering =
403 *static_cast<const SITargetLowering*>(getTargetLowering());
405 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
407 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
408 return glueCopyToOp(N, M0, M0.getValue(1));
411 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
412 unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
413 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
414 if (Subtarget->ldsRequiresM0Init())
415 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
416 } else if (AS == AMDGPUAS::REGION_ADDRESS) {
417 MachineFunction &MF = CurDAG->getMachineFunction();
418 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
420 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
425 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
427 SDNode *Lo = CurDAG->getMachineNode(
428 AMDGPU::S_MOV_B32, DL, MVT::i32,
429 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
431 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
432 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
433 const SDValue Ops[] = {
434 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
435 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
436 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
438 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
441 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
442 EVT VT = N->getValueType(0);
443 unsigned NumVectorElts = VT.getVectorNumElements();
444 EVT EltVT = VT.getVectorElementType();
446 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
448 if (NumVectorElts == 1) {
449 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
454 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
456 // 32 = Max Num Vector Elements
457 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
458 // 1 = Vector Register Class
459 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
461 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
463 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
464 bool IsRegSeq = true;
465 unsigned NOps = N->getNumOperands();
466 for (unsigned i = 0; i < NOps; i++) {
467 // XXX: Why is this here?
468 if (isa<RegisterSDNode>(N->getOperand(i))) {
472 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
473 : R600RegisterInfo::getSubRegFromChannel(i);
474 RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
475 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
477 if (NOps != NumVectorElts) {
478 // Fill in the missing undef elements if this was a scalar_to_vector.
479 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
480 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
482 for (unsigned i = NOps; i < NumVectorElts; ++i) {
483 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
484 : R600RegisterInfo::getSubRegFromChannel(i);
485 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
486 RegSeqArgs[1 + (2 * i) + 1] =
487 CurDAG->getTargetConstant(Sub, DL, MVT::i32);
493 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
496 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
497 unsigned int Opc = N->getOpcode();
498 if (N->isMachineOpcode()) {
500 return; // Already selected.
503 // isa<MemSDNode> almost works but is slightly too permissive for some DS
505 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
506 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
507 Opc == ISD::ATOMIC_LOAD_FADD ||
508 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
509 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
510 N = glueCopyToM0LDSInit(N);
518 // We are selecting i64 ADD here instead of custom lower it during
519 // DAG legalization, so we can fold some i64 ADDs used for address
520 // calculation into the LOAD and STORE instructions.
525 if (N->getValueType(0) != MVT::i64)
528 SelectADD_SUB_I64(N);
533 if (N->getValueType(0) != MVT::i32)
540 SelectUADDO_USUBO(N);
543 case AMDGPUISD::FMUL_W_CHAIN: {
544 SelectFMUL_W_CHAIN(N);
547 case AMDGPUISD::FMA_W_CHAIN: {
548 SelectFMA_W_CHAIN(N);
552 case ISD::SCALAR_TO_VECTOR:
553 case ISD::BUILD_VECTOR: {
554 EVT VT = N->getValueType(0);
555 unsigned NumVectorElts = VT.getVectorNumElements();
556 if (VT.getScalarSizeInBits() == 16) {
557 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
558 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
559 ReplaceNode(N, Packed);
567 assert(VT.getVectorElementType().bitsEq(MVT::i32));
568 unsigned RegClassID =
569 SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
570 SelectBuildVector(N, RegClassID);
573 case ISD::BUILD_PAIR: {
574 SDValue RC, SubReg0, SubReg1;
576 if (N->getValueType(0) == MVT::i128) {
577 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
578 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
579 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
580 } else if (N->getValueType(0) == MVT::i64) {
581 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
582 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
583 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
585 llvm_unreachable("Unhandled value type for BUILD_PAIR");
587 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
588 N->getOperand(1), SubReg1 };
589 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
590 N->getValueType(0), Ops));
595 case ISD::ConstantFP: {
596 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
600 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
601 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
603 ConstantSDNode *C = cast<ConstantSDNode>(N);
604 Imm = C->getZExtValue();
608 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
611 case AMDGPUISD::BFE_I32:
612 case AMDGPUISD::BFE_U32: {
613 // There is a scalar version available, but unlike the vector version which
614 // has a separate operand for the offset and width, the scalar version packs
615 // the width and offset into a single operand. Try to move to the scalar
616 // version if the offsets are constant, so that we can try to keep extended
617 // loads of kernel arguments in SGPRs.
619 // TODO: Technically we could try to pattern match scalar bitshifts of
620 // dynamic values, but it's probably not useful.
621 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
625 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
629 bool Signed = Opc == AMDGPUISD::BFE_I32;
631 uint32_t OffsetVal = Offset->getZExtValue();
632 uint32_t WidthVal = Width->getZExtValue();
634 ReplaceNode(N, getBFE32(Signed, SDLoc(N), N->getOperand(0), OffsetVal,
638 case AMDGPUISD::DIV_SCALE: {
642 case AMDGPUISD::MAD_I64_I32:
643 case AMDGPUISD::MAD_U64_U32: {
649 return SelectMUL_LOHI(N);
650 case ISD::CopyToReg: {
651 const SITargetLowering& Lowering =
652 *static_cast<const SITargetLowering*>(getTargetLowering());
653 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
659 case ISD::SIGN_EXTEND_INREG:
660 if (N->getValueType(0) != MVT::i32)
672 case AMDGPUISD::CVT_PKRTZ_F16_F32:
673 case AMDGPUISD::CVT_PKNORM_I16_F32:
674 case AMDGPUISD::CVT_PKNORM_U16_F32:
675 case AMDGPUISD::CVT_PK_U16_U32:
676 case AMDGPUISD::CVT_PK_I16_I32: {
677 // Hack around using a legal type if f16 is illegal.
678 if (N->getValueType(0) == MVT::i32) {
679 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
680 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
681 { N->getOperand(0), N->getOperand(1) });
688 case ISD::INTRINSIC_W_CHAIN: {
689 SelectINTRINSIC_W_CHAIN(N);
692 case ISD::INTRINSIC_WO_CHAIN: {
693 SelectINTRINSIC_WO_CHAIN(N);
696 case ISD::INTRINSIC_VOID: {
697 SelectINTRINSIC_VOID(N);
705 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
706 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
707 const Instruction *Term = BB->getTerminator();
708 return Term->getMetadata("amdgpu.uniform") ||
709 Term->getMetadata("structurizecfg.uniform");
712 bool AMDGPUDAGToDAGISel::isUnneededShiftMask(const SDNode *N,
713 unsigned ShAmtBits) const {
714 assert(N->getOpcode() == ISD::AND);
716 const APInt &RHS = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue();
717 if (RHS.countTrailingOnes() >= ShAmtBits)
720 const APInt &LHSKnownZeros = CurDAG->computeKnownBits(N->getOperand(0)).Zero;
721 return (LHSKnownZeros | RHS).countTrailingOnes() >= ShAmtBits;
724 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
725 SDValue &N0, SDValue &N1) {
726 if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
727 Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
728 // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
729 // (i64 (bitcast (v2i32 (build_vector
730 // (or (extract_vector_elt V, 0), OFFSET),
731 // (extract_vector_elt V, 1)))))
732 SDValue Lo = Addr.getOperand(0).getOperand(0);
733 if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
734 SDValue BaseLo = Lo.getOperand(0);
735 SDValue BaseHi = Addr.getOperand(0).getOperand(1);
736 // Check that split base (Lo and Hi) are extracted from the same one.
737 if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
738 BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
739 BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
740 // Lo is statically extracted from index 0.
741 isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
742 BaseLo.getConstantOperandVal(1) == 0 &&
743 // Hi is statically extracted from index 0.
744 isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
745 BaseHi.getConstantOperandVal(1) == 1) {
746 N0 = BaseLo.getOperand(0).getOperand(0);
747 N1 = Lo.getOperand(1);
755 bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
756 SDValue &RHS) const {
757 if (CurDAG->isBaseWithConstantOffset(Addr)) {
758 LHS = Addr.getOperand(0);
759 RHS = Addr.getOperand(1);
763 if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, LHS, RHS)) {
764 assert(LHS && RHS && isa<ConstantSDNode>(RHS));
771 StringRef AMDGPUDAGToDAGISel::getPassName() const {
772 return "AMDGPU DAG->DAG Pattern Instruction Selection";
775 //===----------------------------------------------------------------------===//
777 //===----------------------------------------------------------------------===//
779 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
784 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
789 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
790 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
791 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
792 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
793 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
794 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
795 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
796 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
797 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
798 Base = Addr.getOperand(0);
799 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
802 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
808 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
809 const SDLoc &DL) const {
810 SDNode *Mov = CurDAG->getMachineNode(
811 AMDGPU::S_MOV_B32, DL, MVT::i32,
812 CurDAG->getTargetConstant(Val, DL, MVT::i32));
813 return SDValue(Mov, 0);
816 // FIXME: Should only handle addcarry/subcarry
817 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
819 SDValue LHS = N->getOperand(0);
820 SDValue RHS = N->getOperand(1);
822 unsigned Opcode = N->getOpcode();
823 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
825 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
826 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
828 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
829 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
831 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
832 DL, MVT::i32, LHS, Sub0);
833 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
834 DL, MVT::i32, LHS, Sub1);
836 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
837 DL, MVT::i32, RHS, Sub0);
838 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
839 DL, MVT::i32, RHS, Sub1);
841 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
843 static const unsigned OpcMap[2][2][2] = {
844 {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
845 {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
846 {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
847 {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
849 unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
850 unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
854 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
855 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
857 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
858 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
860 SDValue AddHiArgs[] = {
865 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
867 SDValue RegSequenceArgs[] = {
868 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
874 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
875 MVT::i64, RegSequenceArgs);
878 // Replace the carry-use
879 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
882 // Replace the remaining uses.
883 ReplaceNode(N, RegSequence);
886 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
888 SDValue LHS = N->getOperand(0);
889 SDValue RHS = N->getOperand(1);
890 SDValue CI = N->getOperand(2);
892 if (N->isDivergent()) {
893 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
894 : AMDGPU::V_SUBB_U32_e64;
895 CurDAG->SelectNodeTo(
896 N, Opc, N->getVTList(),
898 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
900 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO
901 : AMDGPU::S_SUB_CO_PSEUDO;
902 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
906 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
907 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
908 // carry out despite the _i32 name. These were renamed in VI to _U32.
909 // FIXME: We should probably rename the opcodes here.
910 bool IsAdd = N->getOpcode() == ISD::UADDO;
911 bool IsVALU = N->isDivergent();
913 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
915 if (UI.getUse().getResNo() == 1) {
916 if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) ||
917 (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) {
924 unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
926 CurDAG->SelectNodeTo(
927 N, Opc, N->getVTList(),
928 {N->getOperand(0), N->getOperand(1),
929 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
931 unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
932 : AMDGPU::S_USUBO_PSEUDO;
934 CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
935 {N->getOperand(0), N->getOperand(1)});
939 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
941 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod
944 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
945 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
946 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
947 Ops[8] = N->getOperand(0);
948 Ops[9] = N->getOperand(4);
950 // If there are no source modifiers, prefer fmac over fma because it can use
951 // the smaller VOP2 encoding.
952 bool UseFMAC = Subtarget->hasDLInsts() &&
953 cast<ConstantSDNode>(Ops[0])->isZero() &&
954 cast<ConstantSDNode>(Ops[2])->isZero() &&
955 cast<ConstantSDNode>(Ops[4])->isZero();
956 unsigned Opcode = UseFMAC ? AMDGPU::V_FMAC_F32_e64 : AMDGPU::V_FMA_F32_e64;
957 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), Ops);
960 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
962 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod
965 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
966 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
967 Ops[6] = N->getOperand(0);
968 Ops[7] = N->getOperand(3);
970 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
973 // We need to handle this here because tablegen doesn't support matching
974 // instructions with multiple outputs.
975 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
977 EVT VT = N->getValueType(0);
979 assert(VT == MVT::f32 || VT == MVT::f64);
982 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
984 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
987 SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
988 SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
989 SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
990 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
993 // We need to handle this here because tablegen doesn't support matching
994 // instructions with multiple outputs.
995 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
997 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
999 if (Subtarget->hasMADIntraFwdBug())
1000 Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1001 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1003 Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1005 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1006 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1008 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1011 // We need to handle this here because tablegen doesn't support matching
1012 // instructions with multiple outputs.
1013 void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
1015 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
1017 if (Subtarget->hasMADIntraFwdBug())
1018 Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1019 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1021 Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1023 SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64);
1024 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1025 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp};
1026 SDNode *Mad = CurDAG->getMachineNode(Opc, SL, N->getVTList(), Ops);
1027 if (!SDValue(N, 0).use_empty()) {
1028 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
1029 SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1030 MVT::i32, SDValue(Mad, 0), Sub0);
1031 ReplaceUses(SDValue(N, 0), SDValue(Lo, 0));
1033 if (!SDValue(N, 1).use_empty()) {
1034 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32);
1035 SDNode *Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1036 MVT::i32, SDValue(Mad, 0), Sub1);
1037 ReplaceUses(SDValue(N, 1), SDValue(Hi, 0));
1039 CurDAG->RemoveDeadNode(N);
1042 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1043 if (!isUInt<16>(Offset))
1046 if (!Base || Subtarget->hasUsableDSOffset() ||
1047 Subtarget->unsafeDSOffsetFoldingEnabled())
1050 // On Southern Islands instruction with a negative base value and an offset
1051 // don't seem to work.
1052 return CurDAG->SignBitIsZero(Base);
1055 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1056 SDValue &Offset) const {
1058 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1059 SDValue N0 = Addr.getOperand(0);
1060 SDValue N1 = Addr.getOperand(1);
1061 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1062 if (isDSOffsetLegal(N0, C1->getSExtValue())) {
1065 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1068 } else if (Addr.getOpcode() == ISD::SUB) {
1069 // sub C, x -> add (sub 0, x), C
1070 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1071 int64_t ByteOffset = C->getSExtValue();
1072 if (isDSOffsetLegal(SDValue(), ByteOffset)) {
1073 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1075 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1076 // the known bits in isDSOffsetLegal. We need to emit the selected node
1077 // here, so this is thrown away.
1078 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1079 Zero, Addr.getOperand(1));
1081 if (isDSOffsetLegal(Sub, ByteOffset)) {
1082 SmallVector<SDValue, 3> Opnds;
1083 Opnds.push_back(Zero);
1084 Opnds.push_back(Addr.getOperand(1));
1086 // FIXME: Select to VOP3 version for with-carry.
1087 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1088 if (Subtarget->hasAddNoCarry()) {
1089 SubOp = AMDGPU::V_SUB_U32_e64;
1091 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1094 MachineSDNode *MachineSub =
1095 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1097 Base = SDValue(MachineSub, 0);
1098 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1103 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1104 // If we have a constant address, prefer to put the constant into the
1105 // offset. This can save moves to load the constant address since multiple
1106 // operations can share the zero base address register, and enables merging
1107 // into read2 / write2 instructions.
1111 if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) {
1112 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1113 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1114 DL, MVT::i32, Zero);
1115 Base = SDValue(MovZero, 0);
1116 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1123 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1127 bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1129 unsigned Size) const {
1130 if (Offset0 % Size != 0 || Offset1 % Size != 0)
1132 if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
1135 if (!Base || Subtarget->hasUsableDSOffset() ||
1136 Subtarget->unsafeDSOffsetFoldingEnabled())
1139 // On Southern Islands instruction with a negative base value and an offset
1140 // don't seem to work.
1141 return CurDAG->SignBitIsZero(Base);
1144 // TODO: If offset is too big, put low 16-bit into offset.
1145 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1147 SDValue &Offset1) const {
1148 return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4);
1151 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1153 SDValue &Offset1) const {
1154 return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8);
1157 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1158 SDValue &Offset0, SDValue &Offset1,
1159 unsigned Size) const {
1162 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1163 SDValue N0 = Addr.getOperand(0);
1164 SDValue N1 = Addr.getOperand(1);
1165 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1166 unsigned OffsetValue0 = C1->getZExtValue();
1167 unsigned OffsetValue1 = OffsetValue0 + Size;
1170 if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) {
1172 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1173 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1176 } else if (Addr.getOpcode() == ISD::SUB) {
1177 // sub C, x -> add (sub 0, x), C
1178 if (const ConstantSDNode *C =
1179 dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1180 unsigned OffsetValue0 = C->getZExtValue();
1181 unsigned OffsetValue1 = OffsetValue0 + Size;
1183 if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1185 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1187 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1188 // the known bits in isDSOffsetLegal. We need to emit the selected node
1189 // here, so this is thrown away.
1191 CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1193 if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) {
1194 SmallVector<SDValue, 3> Opnds;
1195 Opnds.push_back(Zero);
1196 Opnds.push_back(Addr.getOperand(1));
1197 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1198 if (Subtarget->hasAddNoCarry()) {
1199 SubOp = AMDGPU::V_SUB_U32_e64;
1201 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1204 MachineSDNode *MachineSub = CurDAG->getMachineNode(
1205 SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds);
1207 Base = SDValue(MachineSub, 0);
1208 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1209 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1214 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1215 unsigned OffsetValue0 = CAddr->getZExtValue();
1216 unsigned OffsetValue1 = OffsetValue0 + Size;
1218 if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1219 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1220 MachineSDNode *MovZero =
1221 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1222 Base = SDValue(MovZero, 0);
1223 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1224 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1232 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1233 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1237 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
1238 SDValue &SOffset, SDValue &Offset,
1239 SDValue &Offen, SDValue &Idxen,
1240 SDValue &Addr64) const {
1241 // Subtarget prefers to use flat instruction
1242 // FIXME: This should be a pattern predicate and not reach here
1243 if (Subtarget->useFlatForGlobal())
1248 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1249 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1250 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1251 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1253 ConstantSDNode *C1 = nullptr;
1255 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1256 C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1257 if (isUInt<32>(C1->getZExtValue()))
1258 N0 = Addr.getOperand(0);
1263 if (N0.getOpcode() == ISD::ADD) {
1264 // (add N2, N3) -> addr64, or
1265 // (add (add N2, N3), C1) -> addr64
1266 SDValue N2 = N0.getOperand(0);
1267 SDValue N3 = N0.getOperand(1);
1268 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1270 if (N2->isDivergent()) {
1271 if (N3->isDivergent()) {
1272 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1273 // addr64, and construct the resource from a 0 address.
1274 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1277 // N2 is divergent, N3 is not.
1282 // N2 is not divergent.
1286 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1287 } else if (N0->isDivergent()) {
1288 // N0 is divergent. Use it as the addr64, and construct the resource from a
1290 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1292 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1295 // (N0 + C1) -> offset
1296 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1302 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1306 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1307 // Legal offset for instruction.
1308 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1312 // Illegal offset, store it in soffset.
1313 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1315 SDValue(CurDAG->getMachineNode(
1316 AMDGPU::S_MOV_B32, DL, MVT::i32,
1317 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1322 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1323 SDValue &VAddr, SDValue &SOffset,
1324 SDValue &Offset) const {
1325 SDValue Ptr, Offen, Idxen, Addr64;
1327 // addr64 bit was removed for volcanic islands.
1328 // FIXME: This should be a pattern predicate and not reach here
1329 if (!Subtarget->hasAddr64())
1332 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1335 ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1336 if (C->getSExtValue()) {
1339 const SITargetLowering& Lowering =
1340 *static_cast<const SITargetLowering*>(getTargetLowering());
1342 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1349 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1352 auto *FI = dyn_cast<FrameIndexSDNode>(N);
1354 FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1356 // We rebase the base address into an absolute stack address and hence
1357 // use constant 0 for soffset. This value must be retained until
1358 // frame elimination and eliminateFrameIndex will choose the appropriate
1359 // frame register if need be.
1360 return std::pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1363 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1364 SDValue Addr, SDValue &Rsrc,
1365 SDValue &VAddr, SDValue &SOffset,
1366 SDValue &ImmOffset) const {
1369 MachineFunction &MF = CurDAG->getMachineFunction();
1370 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1372 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1374 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1375 int64_t Imm = CAddr->getSExtValue();
1376 const int64_t NullPtr =
1377 AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1378 // Don't fold null pointer.
1379 if (Imm != NullPtr) {
1380 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1381 MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1382 AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1383 VAddr = SDValue(MovHighBits, 0);
1385 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1386 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1391 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1394 SDValue N0 = Addr.getOperand(0);
1395 SDValue N1 = Addr.getOperand(1);
1397 // Offsets in vaddr must be positive if range checking is enabled.
1399 // The total computation of vaddr + soffset + offset must not overflow. If
1400 // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1403 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1404 // always perform a range check. If a negative vaddr base index was used,
1405 // this would fail the range check. The overall address computation would
1406 // compute a valid address, but this doesn't happen due to the range
1407 // check. For out-of-bounds MUBUF loads, a 0 is returned.
1409 // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1410 // MUBUF vaddr, but not on older subtargets which can only do this if the
1411 // sign bit is known 0.
1412 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1413 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1414 (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1415 CurDAG->SignBitIsZero(N0))) {
1416 std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1417 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1423 std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1424 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1428 static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
1429 if (Val.getOpcode() != ISD::CopyFromReg)
1431 auto Reg = cast<RegisterSDNode>(Val.getOperand(1))->getReg();
1432 if (!Reg.isPhysical())
1434 auto RC = TRI.getPhysRegBaseClass(Reg);
1435 return RC && TRI.isSGPRClass(RC);
1438 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1442 SDValue &Offset) const {
1443 const SIRegisterInfo *TRI =
1444 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
1445 MachineFunction &MF = CurDAG->getMachineFunction();
1446 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1449 // CopyFromReg <sgpr>
1450 if (IsCopyFromSGPR(*TRI, Addr)) {
1451 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1453 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1457 ConstantSDNode *CAddr;
1458 if (Addr.getOpcode() == ISD::ADD) {
1459 // Add (CopyFromReg <sgpr>) <constant>
1460 CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
1461 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1463 if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0)))
1466 SOffset = Addr.getOperand(0);
1467 } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) &&
1468 SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) {
1470 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1475 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1477 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1481 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1482 SDValue &SOffset, SDValue &Offset
1484 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1485 const SIInstrInfo *TII =
1486 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1488 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1491 if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1492 !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1493 !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1494 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1495 APInt::getAllOnes(32).getZExtValue(); // Size
1498 const SITargetLowering& Lowering =
1499 *static_cast<const SITargetLowering*>(getTargetLowering());
1501 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1507 // Find a load or store from corresponding pattern root.
1508 // Roots may be build_vector, bitconvert or their combinations.
1509 static MemSDNode* findMemSDNode(SDNode *N) {
1510 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1511 if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1513 assert(isa<BuildVectorSDNode>(N));
1514 for (SDValue V : N->op_values())
1516 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1518 llvm_unreachable("cannot find MemSDNode in the pattern!");
1521 bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(SDNode *N, SDValue Addr,
1522 SDValue &VAddr, SDValue &Offset,
1523 uint64_t FlatVariant) const {
1524 int64_t OffsetVal = 0;
1526 unsigned AS = findMemSDNode(N)->getAddressSpace();
1528 bool CanHaveFlatSegmentOffsetBug =
1529 Subtarget->hasFlatSegmentOffsetBug() &&
1530 FlatVariant == SIInstrFlags::FLAT &&
1531 (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS);
1533 if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1535 if (isBaseWithConstantOffset64(Addr, N0, N1)) {
1536 int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1538 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1539 if (TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) {
1541 OffsetVal = COffsetVal;
1543 // If the offset doesn't fit, put the low bits into the offset field and
1546 // For a FLAT instruction the hardware decides whether to access
1547 // global/scratch/shared memory based on the high bits of vaddr,
1548 // ignoring the offset field, so we have to ensure that when we add
1549 // remainder to vaddr it still points into the same underlying object.
1550 // The easiest way to do that is to make sure that we split the offset
1551 // into two pieces that are both >= 0 or both <= 0.
1554 uint64_t RemainderOffset;
1556 std::tie(OffsetVal, RemainderOffset) =
1557 TII->splitFlatOffset(COffsetVal, AS, FlatVariant);
1559 SDValue AddOffsetLo =
1560 getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1561 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1563 if (Addr.getValueType().getSizeInBits() == 32) {
1564 SmallVector<SDValue, 3> Opnds;
1565 Opnds.push_back(N0);
1566 Opnds.push_back(AddOffsetLo);
1567 unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1568 if (Subtarget->hasAddNoCarry()) {
1569 AddOp = AMDGPU::V_ADD_U32_e64;
1570 Opnds.push_back(Clamp);
1572 Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1574 // TODO: Should this try to use a scalar add pseudo if the base address
1575 // is uniform and saddr is usable?
1576 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1577 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1579 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1580 DL, MVT::i32, N0, Sub0);
1581 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1582 DL, MVT::i32, N0, Sub1);
1584 SDValue AddOffsetHi =
1585 getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1587 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1590 CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1591 {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1593 SDNode *Addc = CurDAG->getMachineNode(
1594 AMDGPU::V_ADDC_U32_e64, DL, VTs,
1595 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1597 SDValue RegSequenceArgs[] = {
1598 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1599 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1601 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1602 MVT::i64, RegSequenceArgs),
1610 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
1614 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr,
1616 SDValue &Offset) const {
1617 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FLAT);
1620 bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr,
1622 SDValue &Offset) const {
1623 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FlatGlobal);
1626 bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr,
1628 SDValue &Offset) const {
1629 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
1630 SIInstrFlags::FlatScratch);
1633 // If this matches zero_extend i32:x, return x
1634 static SDValue matchZExtFromI32(SDValue Op) {
1635 if (Op.getOpcode() != ISD::ZERO_EXTEND)
1638 SDValue ExtSrc = Op.getOperand(0);
1639 return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1642 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1643 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1647 SDValue &Offset) const {
1648 int64_t ImmOffset = 0;
1650 // Match the immediate offset first, which canonically is moved as low as
1654 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1655 int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1656 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1658 if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS,
1659 SIInstrFlags::FlatGlobal)) {
1661 ImmOffset = COffsetVal;
1662 } else if (!LHS->isDivergent()) {
1663 if (COffsetVal > 0) {
1665 // saddr + large_offset -> saddr +
1666 // (voffset = large_offset & ~MaxOffset) +
1667 // (large_offset & MaxOffset);
1668 int64_t SplitImmOffset, RemainderOffset;
1669 std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1670 COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, SIInstrFlags::FlatGlobal);
1672 if (isUInt<32>(RemainderOffset)) {
1673 SDNode *VMov = CurDAG->getMachineNode(
1674 AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1675 CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1676 VOffset = SDValue(VMov, 0);
1678 Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
1683 // We are adding a 64 bit SGPR and a constant. If constant bus limit
1684 // is 1 we would need to perform 1 or 2 extra moves for each half of
1685 // the constant and it is better to do a scalar add and then issue a
1686 // single VALU instruction to materialize zero. Otherwise it is less
1687 // instructions to perform VALU adds with immediates or inline literals.
1688 unsigned NumLiterals =
1689 !TII->isInlineConstant(APInt(32, COffsetVal & 0xffffffff)) +
1690 !TII->isInlineConstant(APInt(32, COffsetVal >> 32));
1691 if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
1696 // Match the variable offset.
1697 if (Addr.getOpcode() == ISD::ADD) {
1698 LHS = Addr.getOperand(0);
1699 RHS = Addr.getOperand(1);
1701 if (!LHS->isDivergent()) {
1702 // add (i64 sgpr), (zero_extend (i32 vgpr))
1703 if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1709 if (!SAddr && !RHS->isDivergent()) {
1710 // add (zero_extend (i32 vgpr)), (i64 sgpr)
1711 if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1718 Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1723 if (Addr->isDivergent() || Addr.getOpcode() == ISD::UNDEF ||
1724 isa<ConstantSDNode>(Addr))
1727 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
1728 // moves required to copy a 64-bit SGPR to VGPR.
1731 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32,
1732 CurDAG->getTargetConstant(0, SDLoc(), MVT::i32));
1733 VOffset = SDValue(VMov, 0);
1734 Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1738 static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) {
1739 if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1740 SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1741 } else if (SAddr.getOpcode() == ISD::ADD &&
1742 isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1743 // Materialize this into a scalar move for scalar address to avoid
1745 auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1746 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1747 FI->getValueType(0));
1748 SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, SDLoc(SAddr),
1749 MVT::i32, TFI, SAddr.getOperand(1)),
1756 // Match (32-bit SGPR base) + sext(imm offset)
1757 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
1759 SDValue &Offset) const {
1760 if (Addr->isDivergent())
1765 int64_t COffsetVal = 0;
1767 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1768 COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1769 SAddr = Addr.getOperand(0);
1774 SAddr = SelectSAddrFI(CurDAG, SAddr);
1776 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1778 if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
1779 SIInstrFlags::FlatScratch)) {
1780 int64_t SplitImmOffset, RemainderOffset;
1781 std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1782 COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch);
1784 COffsetVal = SplitImmOffset;
1787 SAddr.getOpcode() == ISD::TargetFrameIndex
1788 ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL)
1789 : CurDAG->getTargetConstant(RemainderOffset, DL, MVT::i32);
1790 SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, DL, MVT::i32,
1795 Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16);
1800 // Check whether the flat scratch SVS swizzle bug affects this access.
1801 bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug(
1802 SDValue VAddr, SDValue SAddr, uint64_t ImmOffset) const {
1803 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
1806 // The bug affects the swizzling of SVS accesses if there is any carry out
1807 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
1808 // voffset to (soffset + inst_offset).
1809 KnownBits VKnown = CurDAG->computeKnownBits(VAddr);
1810 KnownBits SKnown = KnownBits::computeForAddSub(
1811 true, false, CurDAG->computeKnownBits(SAddr),
1812 KnownBits::makeConstant(APInt(32, ImmOffset)));
1813 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
1814 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
1815 return (VMax & 3) + (SMax & 3) >= 4;
1818 bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
1819 SDValue &VAddr, SDValue &SAddr,
1820 SDValue &Offset) const {
1821 int64_t ImmOffset = 0;
1824 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1825 int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1826 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1828 if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
1830 ImmOffset = COffsetVal;
1831 } else if (!LHS->isDivergent() && COffsetVal > 0) {
1833 // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
1834 // (large_offset & MaxOffset);
1835 int64_t SplitImmOffset, RemainderOffset;
1836 std::tie(SplitImmOffset, RemainderOffset)
1837 = TII->splitFlatOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true);
1839 if (isUInt<32>(RemainderOffset)) {
1840 SDNode *VMov = CurDAG->getMachineNode(
1841 AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1842 CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1843 VAddr = SDValue(VMov, 0);
1845 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, SplitImmOffset))
1847 Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
1853 if (Addr.getOpcode() != ISD::ADD)
1856 LHS = Addr.getOperand(0);
1857 RHS = Addr.getOperand(1);
1859 if (!LHS->isDivergent() && RHS->isDivergent()) {
1862 } else if (!RHS->isDivergent() && LHS->isDivergent()) {
1869 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset))
1871 SAddr = SelectSAddrFI(CurDAG, SAddr);
1872 Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1876 // Match an immediate (if Offset is not null) or an SGPR (if SOffset is
1877 // not null) offset. If Imm32Only is true, match only 32-bit immediate
1878 // offsets available on CI.
1879 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1880 SDValue *SOffset, SDValue *Offset,
1881 bool Imm32Only, bool IsBuffer) const {
1882 assert((!SOffset || !Offset) &&
1883 "Cannot match both soffset and offset at the same time!");
1885 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1889 if (ByteOffsetNode.getValueType().isScalarInteger() &&
1890 ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1891 *SOffset = ByteOffsetNode;
1894 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
1895 if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
1896 *SOffset = ByteOffsetNode.getOperand(0);
1903 SDLoc SL(ByteOffsetNode);
1905 // GFX9 and GFX10 have signed byte immediate offsets. The immediate
1906 // offset for S_BUFFER instructions is unsigned.
1907 int64_t ByteOffset = IsBuffer ? C->getZExtValue() : C->getSExtValue();
1908 std::optional<int64_t> EncodedOffset =
1909 AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, IsBuffer);
1910 if (EncodedOffset && Offset && !Imm32Only) {
1911 *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1915 // SGPR and literal offsets are unsigned.
1919 EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
1920 if (EncodedOffset && Offset && Imm32Only) {
1921 *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1925 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
1929 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1931 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
1938 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1939 if (Addr.getValueType() != MVT::i32)
1942 // Zero-extend a 32-bit address.
1945 const MachineFunction &MF = CurDAG->getMachineFunction();
1946 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1947 unsigned AddrHiVal = Info->get32BitAddressHighBits();
1948 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1950 const SDValue Ops[] = {
1951 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1953 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1954 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1956 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1959 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1963 // Match a base and an immediate (if Offset is not null) or an SGPR (if
1964 // SOffset is not null) or an immediate+SGPR offset. If Imm32Only is
1965 // true, match only 32-bit immediate offsets available on CI.
1966 bool AMDGPUDAGToDAGISel::SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase,
1967 SDValue *SOffset, SDValue *Offset,
1969 bool IsBuffer) const {
1970 if (SOffset && Offset) {
1971 assert(!Imm32Only && !IsBuffer);
1973 return SelectSMRDBaseOffset(Addr, B, nullptr, Offset) &&
1974 SelectSMRDBaseOffset(B, SBase, SOffset, nullptr);
1977 // A 32-bit (address + offset) should not cause unsigned 32-bit integer
1978 // wraparound, because s_load instructions perform the addition in 64 bits.
1979 if (Addr.getValueType() == MVT::i32 && Addr.getOpcode() == ISD::ADD &&
1980 !Addr->getFlags().hasNoUnsignedWrap())
1984 // Extract the base and offset if possible.
1985 if (CurDAG->isBaseWithConstantOffset(Addr) || Addr.getOpcode() == ISD::ADD) {
1986 N0 = Addr.getOperand(0);
1987 N1 = Addr.getOperand(1);
1988 } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
1989 assert(N0 && N1 && isa<ConstantSDNode>(N1));
1993 if (SelectSMRDOffset(N1, SOffset, Offset, Imm32Only, IsBuffer)) {
1997 if (SelectSMRDOffset(N0, SOffset, Offset, Imm32Only, IsBuffer)) {
2004 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
2005 SDValue *SOffset, SDValue *Offset,
2006 bool Imm32Only) const {
2007 if (SelectSMRDBaseOffset(Addr, SBase, SOffset, Offset, Imm32Only)) {
2008 SBase = Expand32BitAddress(SBase);
2012 if (Addr.getValueType() == MVT::i32 && Offset && !SOffset) {
2013 SBase = Expand32BitAddress(Addr);
2014 *Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2021 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2022 SDValue &Offset) const {
2023 return SelectSMRD(Addr, SBase, /* SOffset */ nullptr, &Offset);
2026 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2027 SDValue &Offset) const {
2028 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2029 return SelectSMRD(Addr, SBase, /* SOffset */ nullptr, &Offset,
2030 /* Imm32Only */ true);
2033 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2034 SDValue &SOffset) const {
2035 return SelectSMRD(Addr, SBase, &SOffset, /* Offset */ nullptr);
2038 bool AMDGPUDAGToDAGISel::SelectSMRDSgprImm(SDValue Addr, SDValue &SBase,
2040 SDValue &Offset) const {
2041 return SelectSMRD(Addr, SBase, &SOffset, &Offset);
2044 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue N, SDValue &Offset) const {
2045 return SelectSMRDOffset(N, /* SOffset */ nullptr, &Offset,
2046 /* Imm32Only */ false, /* IsBuffer */ true);
2049 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue N,
2050 SDValue &Offset) const {
2051 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2052 return SelectSMRDOffset(N, /* SOffset */ nullptr, &Offset,
2053 /* Imm32Only */ true, /* IsBuffer */ true);
2056 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
2057 SDValue &Offset) const {
2058 // Match the (soffset + offset) pair as a 32-bit register base and
2059 // an immediate offset.
2060 return N.getValueType() == MVT::i32 &&
2061 SelectSMRDBaseOffset(N, /* SBase */ SOffset, /* SOffset*/ nullptr,
2062 &Offset, /* Imm32Only */ false,
2063 /* IsBuffer */ true);
2066 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2068 SDValue &Offset) const {
2071 if (CurDAG->isBaseWithConstantOffset(Index)) {
2072 SDValue N0 = Index.getOperand(0);
2073 SDValue N1 = Index.getOperand(1);
2074 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2077 // Don't peel off the offset (c0) if doing so could possibly lead
2078 // the base (n0) to be negative.
2079 // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2080 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2081 (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2083 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2088 if (isa<ConstantSDNode>(Index))
2092 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2096 SDNode *AMDGPUDAGToDAGISel::getBFE32(bool IsSigned, const SDLoc &DL,
2097 SDValue Val, uint32_t Offset,
2099 if (Val->isDivergent()) {
2100 unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2101 SDValue Off = CurDAG->getTargetConstant(Offset, DL, MVT::i32);
2102 SDValue W = CurDAG->getTargetConstant(Width, DL, MVT::i32);
2104 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, Off, W);
2106 unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2107 // Transformation function, pack the offset and width of a BFE into
2108 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2109 // source, bits [5:0] contain the offset and bits [22:16] the width.
2110 uint32_t PackedVal = Offset | (Width << 16);
2111 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2113 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2116 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2117 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2118 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2119 // Predicate: 0 < b <= c < 32
2121 const SDValue &Shl = N->getOperand(0);
2122 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2123 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2126 uint32_t BVal = B->getZExtValue();
2127 uint32_t CVal = C->getZExtValue();
2129 if (0 < BVal && BVal <= CVal && CVal < 32) {
2130 bool Signed = N->getOpcode() == ISD::SRA;
2131 ReplaceNode(N, getBFE32(Signed, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2139 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2140 switch (N->getOpcode()) {
2142 if (N->getOperand(0).getOpcode() == ISD::SRL) {
2143 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2144 // Predicate: isMask(mask)
2145 const SDValue &Srl = N->getOperand(0);
2146 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2147 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2149 if (Shift && Mask) {
2150 uint32_t ShiftVal = Shift->getZExtValue();
2151 uint32_t MaskVal = Mask->getZExtValue();
2153 if (isMask_32(MaskVal)) {
2154 uint32_t WidthVal = llvm::popcount(MaskVal);
2155 ReplaceNode(N, getBFE32(false, SDLoc(N), Srl.getOperand(0), ShiftVal,
2163 if (N->getOperand(0).getOpcode() == ISD::AND) {
2164 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2165 // Predicate: isMask(mask >> b)
2166 const SDValue &And = N->getOperand(0);
2167 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2168 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2170 if (Shift && Mask) {
2171 uint32_t ShiftVal = Shift->getZExtValue();
2172 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2174 if (isMask_32(MaskVal)) {
2175 uint32_t WidthVal = llvm::popcount(MaskVal);
2176 ReplaceNode(N, getBFE32(false, SDLoc(N), And.getOperand(0), ShiftVal,
2181 } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2182 SelectS_BFEFromShifts(N);
2187 if (N->getOperand(0).getOpcode() == ISD::SHL) {
2188 SelectS_BFEFromShifts(N);
2193 case ISD::SIGN_EXTEND_INREG: {
2194 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2195 SDValue Src = N->getOperand(0);
2196 if (Src.getOpcode() != ISD::SRL)
2199 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2203 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2204 ReplaceNode(N, getBFE32(true, SDLoc(N), Src.getOperand(0),
2205 Amt->getZExtValue(), Width));
2213 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2214 assert(N->getOpcode() == ISD::BRCOND);
2215 if (!N->hasOneUse())
2218 SDValue Cond = N->getOperand(1);
2219 if (Cond.getOpcode() == ISD::CopyToReg)
2220 Cond = Cond.getOperand(2);
2222 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2225 MVT VT = Cond.getOperand(0).getSimpleValueType();
2229 if (VT == MVT::i64) {
2230 auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2232 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2233 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2239 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2240 SDValue Cond = N->getOperand(1);
2242 if (Cond.isUndef()) {
2243 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2244 N->getOperand(2), N->getOperand(0));
2248 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2249 const SIRegisterInfo *TRI = ST->getRegisterInfo();
2251 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2252 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
2253 Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2257 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not
2258 // analyzed what generates the vcc value, so we do not know whether vcc
2259 // bits for disabled lanes are 0. Thus we need to mask out bits for
2262 // For the case that we select S_CBRANCH_SCC1 and it gets
2263 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2264 // SIInstrInfo::moveToVALU which inserts the S_AND).
2266 // We could add an analysis of what generates the vcc value here and omit
2267 // the S_AND when is unnecessary. But it would be better to add a separate
2268 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2269 // catches both cases.
2270 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2271 : AMDGPU::S_AND_B64,
2273 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2280 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2281 CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2282 N->getOperand(2), // Basic Block
2286 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
2287 MVT VT = N->getSimpleValueType(0);
2288 bool IsFMA = N->getOpcode() == ISD::FMA;
2289 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() &&
2290 !Subtarget->hasFmaMixInsts()) ||
2291 ((IsFMA && Subtarget->hasMadMixInsts()) ||
2292 (!IsFMA && Subtarget->hasFmaMixInsts()))) {
2297 SDValue Src0 = N->getOperand(0);
2298 SDValue Src1 = N->getOperand(1);
2299 SDValue Src2 = N->getOperand(2);
2300 unsigned Src0Mods, Src1Mods, Src2Mods;
2302 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand
2303 // using the conversion from f16.
2304 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
2305 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
2306 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
2308 assert((IsFMA || !Mode.allFP32Denormals()) &&
2309 "fmad selected with denormals enabled");
2310 // TODO: We can select this with f32 denormals enabled if all the sources are
2311 // converted from f16 (in which case fmad isn't legal).
2313 if (Sel0 || Sel1 || Sel2) {
2314 // For dummy operands.
2315 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2317 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
2318 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
2319 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
2320 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
2324 CurDAG->SelectNodeTo(N,
2325 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32,
2332 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2333 // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2334 // be copied to an SGPR with readfirstlane.
2335 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2336 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2338 SDValue Chain = N->getOperand(0);
2339 SDValue Ptr = N->getOperand(2);
2340 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2341 MachineMemOperand *MMO = M->getMemOperand();
2342 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2345 if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2346 SDValue PtrBase = Ptr.getOperand(0);
2347 SDValue PtrOffset = Ptr.getOperand(1);
2349 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2350 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) {
2351 N = glueCopyToM0(N, PtrBase);
2352 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2357 N = glueCopyToM0(N, Ptr);
2358 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2363 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2365 N->getOperand(N->getNumOperands() - 1) // New glue
2368 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2369 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2372 // We need to handle this here because tablegen doesn't support matching
2373 // instructions with multiple outputs.
2374 void AMDGPUDAGToDAGISel::SelectDSBvhStackIntrinsic(SDNode *N) {
2375 unsigned Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2376 SDValue Ops[] = {N->getOperand(2), N->getOperand(3), N->getOperand(4),
2377 N->getOperand(5), N->getOperand(0)};
2379 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2380 MachineMemOperand *MMO = M->getMemOperand();
2381 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2382 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2385 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2387 case Intrinsic::amdgcn_ds_gws_init:
2388 return AMDGPU::DS_GWS_INIT;
2389 case Intrinsic::amdgcn_ds_gws_barrier:
2390 return AMDGPU::DS_GWS_BARRIER;
2391 case Intrinsic::amdgcn_ds_gws_sema_v:
2392 return AMDGPU::DS_GWS_SEMA_V;
2393 case Intrinsic::amdgcn_ds_gws_sema_br:
2394 return AMDGPU::DS_GWS_SEMA_BR;
2395 case Intrinsic::amdgcn_ds_gws_sema_p:
2396 return AMDGPU::DS_GWS_SEMA_P;
2397 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2398 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2400 llvm_unreachable("not a gws intrinsic");
2404 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2405 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2406 !Subtarget->hasGWSSemaReleaseAll()) {
2412 // Chain, intrinsic ID, vsrc, offset
2413 const bool HasVSrc = N->getNumOperands() == 4;
2414 assert(HasVSrc || N->getNumOperands() == 3);
2417 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2419 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2420 MachineMemOperand *MMO = M->getMemOperand();
2422 // Don't worry if the offset ends up in a VGPR. Only one lane will have
2423 // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2425 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2426 // offset field) % 64. Some versions of the programming guide omit the m0
2427 // part, or claim it's from offset 0.
2428 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2429 // If we have a constant offset, try to use the 0 in m0 as the base.
2430 // TODO: Look into changing the default m0 initialization value. If the
2431 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2432 // the immediate offset.
2433 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2434 ImmOffset = ConstOffset->getZExtValue();
2436 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2437 ImmOffset = BaseOffset.getConstantOperandVal(1);
2438 BaseOffset = BaseOffset.getOperand(0);
2441 // Prefer to do the shift in an SGPR since it should be possible to use m0
2442 // as the result directly. If it's already an SGPR, it will be eliminated
2445 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2447 // Shift to offset in m0
2449 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2450 SDValue(SGPROffset, 0),
2451 CurDAG->getTargetConstant(16, SL, MVT::i32));
2452 glueCopyToM0(N, SDValue(M0Base, 0));
2455 SDValue Chain = N->getOperand(0);
2456 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2458 const unsigned Opc = gwsIntrinToOpcode(IntrID);
2459 SmallVector<SDValue, 5> Ops;
2461 Ops.push_back(N->getOperand(2));
2462 Ops.push_back(OffsetField);
2463 Ops.push_back(Chain);
2465 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2466 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2469 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2470 if (Subtarget->getLDSBankCount() != 16) {
2471 // This is a single instruction with a pattern.
2478 // This requires 2 instructions. It is possible to write a pattern to support
2479 // this, but the generated isel emitter doesn't correctly deal with multiple
2480 // output instructions using the same physical register input. The copy to m0
2481 // is incorrectly placed before the second instruction.
2483 // TODO: Match source modifiers.
2486 // (int_amdgcn_interp_p1_f16
2487 // (VOP3Mods f32:$src0, i32:$src0_modifiers),
2488 // (i32 timm:$attrchan), (i32 timm:$attr),
2489 // (i1 timm:$high), M0),
2490 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2491 // timm:$attrchan, 0,
2492 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2493 // let Predicates = [has16BankLDS];
2497 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2498 N->getOperand(5), SDValue());
2500 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2503 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2504 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2505 N->getOperand(3), // Attr
2506 N->getOperand(2), // Attrchan
2507 ToM0.getValue(1) // In glue
2510 SDNode *InterpP1LV =
2511 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2512 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2513 N->getOperand(1), // Src0
2514 N->getOperand(3), // Attr
2515 N->getOperand(2), // Attrchan
2516 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2517 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2518 N->getOperand(4), // high
2519 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2520 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2521 SDValue(InterpMov, 1)
2524 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2527 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2528 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2530 case Intrinsic::amdgcn_ds_append:
2531 case Intrinsic::amdgcn_ds_consume: {
2532 if (N->getValueType(0) != MVT::i32)
2534 SelectDSAppendConsume(N, IntrID);
2537 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2538 SelectDSBvhStackIntrinsic(N);
2545 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2546 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2549 case Intrinsic::amdgcn_wqm:
2550 Opcode = AMDGPU::WQM;
2552 case Intrinsic::amdgcn_softwqm:
2553 Opcode = AMDGPU::SOFT_WQM;
2555 case Intrinsic::amdgcn_wwm:
2556 case Intrinsic::amdgcn_strict_wwm:
2557 Opcode = AMDGPU::STRICT_WWM;
2559 case Intrinsic::amdgcn_strict_wqm:
2560 Opcode = AMDGPU::STRICT_WQM;
2562 case Intrinsic::amdgcn_interp_p1_f16:
2563 SelectInterpP1F16(N);
2570 SDValue Src = N->getOperand(1);
2571 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2574 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2575 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2577 case Intrinsic::amdgcn_ds_gws_init:
2578 case Intrinsic::amdgcn_ds_gws_barrier:
2579 case Intrinsic::amdgcn_ds_gws_sema_v:
2580 case Intrinsic::amdgcn_ds_gws_sema_br:
2581 case Intrinsic::amdgcn_ds_gws_sema_p:
2582 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2583 SelectDS_GWS(N, IntrID);
2592 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2594 bool AllowAbs) const {
2598 if (Src.getOpcode() == ISD::FNEG) {
2599 Mods |= SISrcMods::NEG;
2600 Src = Src.getOperand(0);
2603 if (AllowAbs && Src.getOpcode() == ISD::FABS) {
2604 Mods |= SISrcMods::ABS;
2605 Src = Src.getOperand(0);
2611 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2612 SDValue &SrcMods) const {
2614 if (SelectVOP3ModsImpl(In, Src, Mods)) {
2615 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2622 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
2623 SDValue &SrcMods) const {
2625 if (SelectVOP3ModsImpl(In, Src, Mods, /* AllowAbs */ false)) {
2626 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2633 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2634 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2641 bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
2645 if (SelectVOP3ModsImpl(In, Src, Mods, /* AllowAbs */ false)) {
2647 Mods |= SISrcMods::OP_SEL_0;
2648 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2655 bool AMDGPUDAGToDAGISel::SelectVINTERPMods(SDValue In, SDValue &Src,
2656 SDValue &SrcMods) const {
2657 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ false);
2660 bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(SDValue In, SDValue &Src,
2661 SDValue &SrcMods) const {
2662 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ true);
2665 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2666 SDValue &SrcMods, SDValue &Clamp,
2667 SDValue &Omod) const {
2669 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2670 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2672 return SelectVOP3Mods(In, Src, SrcMods);
2675 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
2676 SDValue &SrcMods, SDValue &Clamp,
2677 SDValue &Omod) const {
2679 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2680 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2682 return SelectVOP3BMods(In, Src, SrcMods);
2685 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2686 SDValue &Clamp, SDValue &Omod) const {
2690 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2691 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2696 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2697 SDValue &SrcMods, bool IsDOT) const {
2701 if (Src.getOpcode() == ISD::FNEG) {
2702 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2703 Src = Src.getOperand(0);
2706 if (Src.getOpcode() == ISD::BUILD_VECTOR && Src.getNumOperands() == 2 &&
2707 (!IsDOT || !Subtarget->hasDOTOpSelHazard())) {
2708 unsigned VecMods = Mods;
2710 SDValue Lo = stripBitcast(Src.getOperand(0));
2711 SDValue Hi = stripBitcast(Src.getOperand(1));
2713 if (Lo.getOpcode() == ISD::FNEG) {
2714 Lo = stripBitcast(Lo.getOperand(0));
2715 Mods ^= SISrcMods::NEG;
2718 if (Hi.getOpcode() == ISD::FNEG) {
2719 Hi = stripBitcast(Hi.getOperand(0));
2720 Mods ^= SISrcMods::NEG_HI;
2723 if (isExtractHiElt(Lo, Lo))
2724 Mods |= SISrcMods::OP_SEL_0;
2726 if (isExtractHiElt(Hi, Hi))
2727 Mods |= SISrcMods::OP_SEL_1;
2729 unsigned VecSize = Src.getValueSizeInBits();
2730 Lo = stripExtractLoElt(Lo);
2731 Hi = stripExtractLoElt(Hi);
2733 if (Lo.getValueSizeInBits() > VecSize) {
2734 Lo = CurDAG->getTargetExtractSubreg(
2735 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2736 MVT::getIntegerVT(VecSize), Lo);
2739 if (Hi.getValueSizeInBits() > VecSize) {
2740 Hi = CurDAG->getTargetExtractSubreg(
2741 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2742 MVT::getIntegerVT(VecSize), Hi);
2745 assert(Lo.getValueSizeInBits() <= VecSize &&
2746 Hi.getValueSizeInBits() <= VecSize);
2748 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2749 // Really a scalar input. Just select from the low half of the register to
2752 if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) {
2755 assert(Lo.getValueSizeInBits() == 32 && VecSize == 64);
2758 SDValue Undef = SDValue(
2759 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
2760 Lo.getValueType()), 0);
2761 auto RC = Lo->isDivergent() ? AMDGPU::VReg_64RegClassID
2762 : AMDGPU::SReg_64RegClassID;
2763 const SDValue Ops[] = {
2764 CurDAG->getTargetConstant(RC, SL, MVT::i32),
2765 Lo, CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2766 Undef, CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32) };
2768 Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
2769 Src.getValueType(), Ops), 0);
2771 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2775 if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) {
2776 uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF()
2777 .bitcastToAPInt().getZExtValue();
2778 if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) {
2779 Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);;
2780 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2788 // Packed instructions do not have abs modifiers.
2789 Mods |= SISrcMods::OP_SEL_1;
2791 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2795 bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(SDValue In, SDValue &Src,
2796 SDValue &SrcMods) const {
2797 return SelectVOP3PMods(In, Src, SrcMods, true);
2800 bool AMDGPUDAGToDAGISel::SelectDotIUVOP3PMods(SDValue In, SDValue &Src) const {
2801 const ConstantSDNode *C = cast<ConstantSDNode>(In);
2802 // Literal i1 value set in intrinsic, represents SrcMods for the next operand.
2803 // 1 promotes packed values to signed, 0 treats them as unsigned.
2804 assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
2806 unsigned Mods = SISrcMods::OP_SEL_1;
2807 unsigned SrcSign = C->getAPIntValue().getZExtValue();
2809 Mods ^= SISrcMods::NEG;
2811 Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2815 bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(SDValue In,
2816 SDValue &Src) const {
2817 const ConstantSDNode *C = cast<ConstantSDNode>(In);
2818 assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
2820 unsigned Mods = SISrcMods::OP_SEL_1;
2821 unsigned SrcVal = C->getAPIntValue().getZExtValue();
2823 Mods |= SISrcMods::OP_SEL_0;
2825 Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2829 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2830 SDValue &SrcMods) const {
2832 // FIXME: Handle op_sel
2833 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2837 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2838 SDValue &SrcMods) const {
2839 // FIXME: Handle op_sel
2840 return SelectVOP3Mods(In, Src, SrcMods);
2843 // The return value is not whether the match is possible (which it always is),
2844 // but whether or not it a conversion is really used.
2845 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2846 unsigned &Mods) const {
2848 SelectVOP3ModsImpl(In, Src, Mods);
2850 if (Src.getOpcode() == ISD::FP_EXTEND) {
2851 Src = Src.getOperand(0);
2852 assert(Src.getValueType() == MVT::f16);
2853 Src = stripBitcast(Src);
2855 // Be careful about folding modifiers if we already have an abs. fneg is
2856 // applied last, so we don't want to apply an earlier fneg.
2857 if ((Mods & SISrcMods::ABS) == 0) {
2859 SelectVOP3ModsImpl(Src, Src, ModsTmp);
2861 if ((ModsTmp & SISrcMods::NEG) != 0)
2862 Mods ^= SISrcMods::NEG;
2864 if ((ModsTmp & SISrcMods::ABS) != 0)
2865 Mods |= SISrcMods::ABS;
2868 // op_sel/op_sel_hi decide the source type and source.
2869 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2870 // If the sources's op_sel is set, it picks the high half of the source
2873 Mods |= SISrcMods::OP_SEL_1;
2874 if (isExtractHiElt(Src, Src)) {
2875 Mods |= SISrcMods::OP_SEL_0;
2877 // TODO: Should we try to look for neg/abs here?
2886 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2887 SDValue &SrcMods) const {
2889 SelectVOP3PMadMixModsImpl(In, Src, Mods);
2890 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2894 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2896 return CurDAG->getUNDEF(MVT::i32);
2898 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2900 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2903 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2905 return CurDAG->getConstant(
2906 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2910 if (isExtractHiElt(In, Src))
2916 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2917 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2919 const SIRegisterInfo *SIRI =
2920 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2921 const SIInstrInfo * SII =
2922 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2925 bool AllUsesAcceptSReg = true;
2926 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2927 Limit < 10 && U != E; ++U, ++Limit) {
2928 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2930 // If the register class is unknown, it could be an unknown
2931 // register class that needs to be an SGPR, e.g. an inline asm
2933 if (!RC || SIRI->isSGPRClass(RC))
2936 if (RC != &AMDGPU::VS_32RegClass) {
2937 AllUsesAcceptSReg = false;
2939 if (User->isMachineOpcode()) {
2940 unsigned Opc = User->getMachineOpcode();
2941 const MCInstrDesc &Desc = SII->get(Opc);
2942 if (Desc.isCommutable()) {
2943 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2944 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2945 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2946 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2947 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2948 if (CommutedRC == &AMDGPU::VS_32RegClass)
2949 AllUsesAcceptSReg = true;
2953 // If "AllUsesAcceptSReg == false" so far we haven't succeeded
2954 // commuting current user. This means have at least one use
2955 // that strictly require VGPR. Thus, we will not attempt to commute
2956 // other user instructions.
2957 if (!AllUsesAcceptSReg)
2961 return !AllUsesAcceptSReg && (Limit < 10);
2964 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2965 auto Ld = cast<LoadSDNode>(N);
2967 if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(Ld->getMemOperand()))
2970 return Ld->getAlign() >= Align(4) &&
2971 ((Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2972 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
2973 (Subtarget->getScalarizeGlobalBehavior() &&
2974 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2976 static_cast<const SITargetLowering *>(getTargetLowering())
2977 ->isMemOpHasNoClobberedMemOperand(N)));
2980 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
2981 const AMDGPUTargetLowering& Lowering =
2982 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
2983 bool IsModified = false;
2987 // Go over all selected nodes and try to fold them a bit more
2988 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
2989 while (Position != CurDAG->allnodes_end()) {
2990 SDNode *Node = &*Position++;
2991 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
2995 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
2996 if (ResNode != Node) {
2998 ReplaceUses(Node, ResNode);
3002 CurDAG->RemoveDeadNodes();
3003 } while (IsModified);
3006 char AMDGPUDAGToDAGISel::ID = 0;