1 //===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// This file implements the targeting of the RegisterBankInfo class for
14 /// AMDGPU has unique register bank constraints that require special high level
15 /// strategies to deal with. There are two main true physical register banks
16 /// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a
17 /// sort of pseudo-register bank needed to represent SGPRs used in a vector
18 /// boolean context. There is also the AGPR bank, which is a special purpose
19 /// physical register bank present on some subtargets.
21 /// Copying from VGPR to SGPR is generally illegal, unless the value is known to
22 /// be uniform. It is generally not valid to legalize operands by inserting
23 /// copies as on other targets. Operations which require uniform, SGPR operands
24 /// generally require scalarization by repeatedly executing the instruction,
25 /// activating each set of lanes using a unique set of input values. This is
26 /// referred to as a waterfall loop.
30 /// Booleans (s1 values) requires special consideration. A vector compare result
31 /// is naturally a bitmask with one bit per lane, in a 32 or 64-bit
32 /// register. These are represented with the VCC bank. During selection, we need
33 /// to be able to unambiguously go back from a register class to a register
34 /// bank. To distinguish whether an SGPR should use the SGPR or VCC register
35 /// bank, we need to know the use context type. An SGPR s1 value always means a
36 /// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets
37 /// SCC, which is a 1-bit unaddressable register. This will need to be copied to
38 /// a 32-bit virtual register. Taken together, this means we need to adjust the
39 /// type of boolean operations to be regbank legal. All SALU booleans need to be
40 /// widened to 32-bits, and all VALU booleans need to be s1 values.
42 /// A noteworthy exception to the s1-means-vcc rule is for legalization artifact
43 /// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc
44 /// bank. A non-boolean source (such as a truncate from a 1-bit load from
45 /// memory) will require a copy to the VCC bank which will require clearing the
46 /// high bits and inserting a compare.
48 /// \par Constant bus restriction
50 /// VALU instructions have a limitation known as the constant bus
51 /// restriction. Most VALU instructions can use SGPR operands, but may read at
52 /// most 1 SGPR or constant literal value (this to 2 in gfx10 for most
53 /// instructions). This is one unique SGPR, so the same SGPR may be used for
54 /// multiple operands. From a register bank perspective, any combination of
55 /// operands should be legal as an SGPR, but this is contextually dependent on
56 /// the SGPR operands all being the same register. There is therefore optimal to
57 /// choose the SGPR with the most uses to minimize the number of copies.
59 /// We avoid trying to solve this problem in RegBankSelect. Any VALU G_*
60 /// operation should have its source operands all mapped to VGPRs (except for
61 /// VCC), inserting copies from any SGPR operands. This the most trival legal
62 /// mapping. Anything beyond the simplest 1:1 instruction selection would be too
63 /// complicated to solve here. Every optimization pattern or instruction
64 /// selected to multiple outputs would have to enforce this rule, and there
65 /// would be additional complexity in tracking this rule for every G_*
66 /// operation. By forcing all inputs to VGPRs, it also simplifies the task of
67 /// picking the optimal operand combination from a post-isel optimization pass.
69 //===----------------------------------------------------------------------===//
71 #include "AMDGPURegisterBankInfo.h"
73 #include "AMDGPUGlobalISelUtils.h"
74 #include "AMDGPUInstrInfo.h"
75 #include "AMDGPUSubtarget.h"
76 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
77 #include "SIMachineFunctionInfo.h"
78 #include "SIRegisterInfo.h"
79 #include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
80 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
81 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
82 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
83 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
84 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
85 #include "llvm/CodeGen/TargetRegisterInfo.h"
86 #include "llvm/CodeGen/TargetSubtargetInfo.h"
87 #include "llvm/IR/Constants.h"
89 #define GET_TARGET_REGBANK_IMPL
90 #include "AMDGPUGenRegisterBank.inc"
92 // This file will be TableGen'ed at some point.
93 #include "AMDGPUGenRegisterBankInfo.def"
96 using namespace MIPatternMatch;
100 // Observer to apply a register bank to new registers created by LegalizerHelper.
101 class ApplyRegBankMapping final : public GISelChangeObserver {
103 const AMDGPURegisterBankInfo &RBI;
104 MachineRegisterInfo &MRI;
105 const RegisterBank *NewBank;
106 SmallVector<MachineInstr *, 4> NewInsts;
109 ApplyRegBankMapping(const AMDGPURegisterBankInfo &RBI_,
110 MachineRegisterInfo &MRI_, const RegisterBank *RB)
111 : RBI(RBI_), MRI(MRI_), NewBank(RB) {}
113 ~ApplyRegBankMapping() {
114 for (MachineInstr *MI : NewInsts)
118 /// Set any registers that don't have a set register class or bank to SALU.
119 void applyBank(MachineInstr &MI) {
120 const unsigned Opc = MI.getOpcode();
121 if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT ||
122 Opc == AMDGPU::G_SEXT) {
123 // LegalizerHelper wants to use the basic legalization artifacts when
124 // widening etc. We don't handle selection with vcc in artifact sources,
125 // so we need to use a sslect instead to handle these properly.
126 Register DstReg = MI.getOperand(0).getReg();
127 Register SrcReg = MI.getOperand(1).getReg();
128 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, *RBI.TRI);
129 if (SrcBank == &AMDGPU::VCCRegBank) {
130 const LLT S32 = LLT::scalar(32);
131 assert(MRI.getType(SrcReg) == LLT::scalar(1));
132 assert(MRI.getType(DstReg) == S32);
133 assert(NewBank == &AMDGPU::VGPRRegBank);
135 // Replace the extension with a select, which really uses the boolean
137 MachineIRBuilder B(MI);
138 auto True = B.buildConstant(S32, Opc == AMDGPU::G_SEXT ? -1 : 1);
139 auto False = B.buildConstant(S32, 0);
140 B.buildSelect(DstReg, SrcReg, True, False);
141 MRI.setRegBank(True.getReg(0), *NewBank);
142 MRI.setRegBank(False.getReg(0), *NewBank);
143 MI.eraseFromParent();
146 assert(!MRI.getRegClassOrRegBank(DstReg));
147 MRI.setRegBank(DstReg, *NewBank);
152 if (Opc == AMDGPU::G_TRUNC) {
153 Register DstReg = MI.getOperand(0).getReg();
154 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI);
155 assert(DstBank != &AMDGPU::VCCRegBank);
159 for (MachineOperand &Op : MI.operands()) {
163 // We may see physical registers if building a real MI
164 Register Reg = Op.getReg();
165 if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg))
168 const RegisterBank *RB = NewBank;
169 if (MRI.getType(Reg) == LLT::scalar(1)) {
170 assert(NewBank == &AMDGPU::VGPRRegBank &&
171 "s1 operands should only be used for vector bools");
172 assert((MI.getOpcode() != AMDGPU::G_TRUNC &&
173 MI.getOpcode() != AMDGPU::G_ANYEXT) &&
174 "not expecting legalization artifacts here");
175 RB = &AMDGPU::VCCRegBank;
178 MRI.setRegBank(Reg, *RB);
182 void erasingInstr(MachineInstr &MI) override {}
184 void createdInstr(MachineInstr &MI) override {
185 // At this point, the instruction was just inserted and has no operands.
186 NewInsts.push_back(&MI);
189 void changingInstr(MachineInstr &MI) override {}
190 void changedInstr(MachineInstr &MI) override {}
194 AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
195 : AMDGPUGenRegisterBankInfo(),
197 TRI(Subtarget.getRegisterInfo()),
198 TII(Subtarget.getInstrInfo()) {
200 // HACK: Until this is fully tablegen'd.
201 static llvm::once_flag InitializeRegisterBankFlag;
203 static auto InitializeRegisterBankOnce = [this]() {
204 assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank &&
205 &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank &&
206 &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank);
210 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);
213 static bool isVectorRegisterBank(const RegisterBank &Bank) {
214 unsigned BankID = Bank.getID();
215 return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID;
218 unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
219 const RegisterBank &Src,
220 unsigned Size) const {
221 // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane?
222 if (Dst.getID() == AMDGPU::SGPRRegBankID &&
223 (isVectorRegisterBank(Src) || Src.getID() == AMDGPU::VCCRegBankID)) {
224 return std::numeric_limits<unsigned>::max();
227 // Bool values are tricky, because the meaning is based on context. The SCC
228 // and VCC banks are for the natural scalar and vector conditions produced by
231 // Legalization doesn't know about the necessary context, so an s1 use may
232 // have been a truncate from an arbitrary value, in which case a copy (lowered
233 // as a compare with 0) needs to be inserted.
235 (Dst.getID() == AMDGPU::SGPRRegBankID) &&
236 (isVectorRegisterBank(Src) ||
237 Src.getID() == AMDGPU::SGPRRegBankID ||
238 Src.getID() == AMDGPU::VCCRegBankID))
239 return std::numeric_limits<unsigned>::max();
241 // There is no direct copy between AGPRs.
242 if (Dst.getID() == AMDGPU::AGPRRegBankID &&
243 Src.getID() == AMDGPU::AGPRRegBankID)
246 return RegisterBankInfo::copyCost(Dst, Src, Size);
249 unsigned AMDGPURegisterBankInfo::getBreakDownCost(
250 const ValueMapping &ValMapping,
251 const RegisterBank *CurBank) const {
252 // Check if this is a breakdown for G_LOAD to move the pointer from SGPR to
254 // FIXME: Is there a better way to do this?
255 if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64)
256 return 10; // This is expensive.
258 assert(ValMapping.NumBreakDowns == 2 &&
259 ValMapping.BreakDown[0].Length == 32 &&
260 ValMapping.BreakDown[0].StartIdx == 0 &&
261 ValMapping.BreakDown[1].Length == 32 &&
262 ValMapping.BreakDown[1].StartIdx == 32 &&
263 ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank);
265 // 32-bit extract of a 64-bit value is just access of a subregister, so free.
266 // TODO: Cost of 0 hits assert, though it's not clear it's what we really
269 // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
270 // alignment restrictions, but this probably isn't important.
275 AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
277 if (&RC == &AMDGPU::SReg_1RegClass)
278 return AMDGPU::VCCRegBank;
280 // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a
282 if (TRI->isSGPRClass(&RC)) {
283 // FIXME: This probably came from a copy from a physical register, which
284 // should be inferrrable from the copied to-type. We don't have many boolean
285 // physical register constraints so just assume a normal SGPR for now.
287 return AMDGPU::SGPRRegBank;
289 return Ty == LLT::scalar(1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
292 return TRI->isAGPRClass(&RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank;
295 template <unsigned NumOps>
296 RegisterBankInfo::InstructionMappings
297 AMDGPURegisterBankInfo::addMappingFromTable(
298 const MachineInstr &MI, const MachineRegisterInfo &MRI,
299 const std::array<unsigned, NumOps> RegSrcOpIdx,
300 ArrayRef<OpRegBankEntry<NumOps>> Table) const {
302 InstructionMappings AltMappings;
304 SmallVector<const ValueMapping *, 10> Operands(MI.getNumOperands());
306 unsigned Sizes[NumOps];
307 for (unsigned I = 0; I < NumOps; ++I) {
308 Register Reg = MI.getOperand(RegSrcOpIdx[I]).getReg();
309 Sizes[I] = getSizeInBits(Reg, MRI, *TRI);
312 for (unsigned I = 0, E = MI.getNumExplicitDefs(); I != E; ++I) {
313 unsigned SizeI = getSizeInBits(MI.getOperand(I).getReg(), MRI, *TRI);
314 Operands[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SizeI);
317 // getInstrMapping's default mapping uses ID 1, so start at 2.
318 unsigned MappingID = 2;
319 for (const auto &Entry : Table) {
320 for (unsigned I = 0; I < NumOps; ++I) {
321 int OpIdx = RegSrcOpIdx[I];
322 Operands[OpIdx] = AMDGPU::getValueMapping(Entry.RegBanks[I], Sizes[I]);
325 AltMappings.push_back(&getInstructionMapping(MappingID++, Entry.Cost,
326 getOperandsMapping(Operands),
333 RegisterBankInfo::InstructionMappings
334 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(
335 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
336 switch (MI.getIntrinsicID()) {
337 case Intrinsic::amdgcn_readlane: {
338 static const OpRegBankEntry<3> Table[2] = {
340 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 },
342 // Need a readfirstlane for the index.
343 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }
346 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } };
347 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
349 case Intrinsic::amdgcn_writelane: {
350 static const OpRegBankEntry<4> Table[4] = {
352 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 },
354 // Need readfirstlane of first op
355 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 },
357 // Need readfirstlane of second op
358 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 },
360 // Need readfirstlane of both ops
361 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 3 }
364 // rsrc, voffset, offset
365 const std::array<unsigned, 4> RegSrcOpIdx = { { 0, 2, 3, 4 } };
366 return addMappingFromTable<4>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
369 return RegisterBankInfo::getInstrAlternativeMappings(MI);
373 RegisterBankInfo::InstructionMappings
374 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(
375 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
377 switch (MI.getIntrinsicID()) {
378 case Intrinsic::amdgcn_s_buffer_load: {
379 static const OpRegBankEntry<2> Table[4] = {
381 { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 },
383 // Only need 1 register in loop
384 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 300 },
386 // Have to waterfall the resource.
387 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1000 },
389 // Have to waterfall the resource, and the offset.
390 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1500 }
394 const std::array<unsigned, 2> RegSrcOpIdx = { { 2, 3 } };
395 return addMappingFromTable<2>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
397 case Intrinsic::amdgcn_ds_ordered_add:
398 case Intrinsic::amdgcn_ds_ordered_swap: {
400 static const OpRegBankEntry<3> Table[2] = {
402 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 },
404 // Need a readfirstlane for m0
405 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }
408 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } };
409 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
411 case Intrinsic::amdgcn_s_sendmsg:
412 case Intrinsic::amdgcn_s_sendmsghalt: {
413 // FIXME: Should have no register for immediate
414 static const OpRegBankEntry<1> Table[2] = {
416 { { AMDGPU::SGPRRegBankID }, 1 },
419 { { AMDGPU::VGPRRegBankID }, 3 }
422 const std::array<unsigned, 1> RegSrcOpIdx = { { 2 } };
423 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
426 return RegisterBankInfo::getInstrAlternativeMappings(MI);
430 static bool memOpHasNoClobbered(const MachineMemOperand *MMO) {
431 const Instruction *I = dyn_cast_or_null<Instruction>(MMO->getValue());
432 return I && I->getMetadata("amdgpu.noclobber");
435 // FIXME: Returns uniform if there's no source value information. This is
437 static bool isScalarLoadLegal(const MachineInstr &MI) {
438 if (!MI.hasOneMemOperand())
441 const MachineMemOperand *MMO = *MI.memoperands_begin();
442 const unsigned AS = MMO->getAddrSpace();
443 const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS ||
444 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
446 // There are no extending SMRD/SMEM loads, and they require 4-byte alignment.
447 return MMO->getSize() >= 4 && MMO->getAlign() >= Align(4) &&
448 // Can't do a scalar atomic load.
450 // Don't use scalar loads for volatile accesses to non-constant address
452 (IsConst || !MMO->isVolatile()) &&
453 // Memory must be known constant, or not written before this load.
454 (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) &&
455 AMDGPUInstrInfo::isUniformMMO(MMO);
458 RegisterBankInfo::InstructionMappings
459 AMDGPURegisterBankInfo::getInstrAlternativeMappings(
460 const MachineInstr &MI) const {
462 const MachineFunction &MF = *MI.getParent()->getParent();
463 const MachineRegisterInfo &MRI = MF.getRegInfo();
466 InstructionMappings AltMappings;
467 switch (MI.getOpcode()) {
468 case TargetOpcode::G_CONSTANT: {
469 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
471 static const OpRegBankEntry<1> Table[3] = {
472 { { AMDGPU::VGPRRegBankID }, 1 },
473 { { AMDGPU::SGPRRegBankID }, 1 },
474 { { AMDGPU::VCCRegBankID }, 1 }
477 return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table);
482 case TargetOpcode::G_FCONSTANT:
483 case TargetOpcode::G_FRAME_INDEX:
484 case TargetOpcode::G_GLOBAL_VALUE: {
485 static const OpRegBankEntry<1> Table[2] = {
486 { { AMDGPU::VGPRRegBankID }, 1 },
487 { { AMDGPU::SGPRRegBankID }, 1 }
490 return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table);
492 case TargetOpcode::G_AND:
493 case TargetOpcode::G_OR:
494 case TargetOpcode::G_XOR: {
495 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
498 // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0.
499 const InstructionMapping &SCCMapping = getInstructionMapping(
500 1, 1, getOperandsMapping(
501 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32),
502 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32),
503 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32)}),
505 AltMappings.push_back(&SCCMapping);
507 const InstructionMapping &VCCMapping0 = getInstructionMapping(
508 2, 1, getOperandsMapping(
509 {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size),
510 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size),
511 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size)}),
513 AltMappings.push_back(&VCCMapping0);
520 const InstructionMapping &SSMapping = getInstructionMapping(
521 1, 1, getOperandsMapping(
522 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
523 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
524 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
526 AltMappings.push_back(&SSMapping);
528 const InstructionMapping &VVMapping = getInstructionMapping(
529 2, 2, getOperandsMapping(
530 {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
531 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
532 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}),
534 AltMappings.push_back(&VVMapping);
537 case TargetOpcode::G_LOAD:
538 case TargetOpcode::G_ZEXTLOAD:
539 case TargetOpcode::G_SEXTLOAD: {
540 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
541 LLT PtrTy = MRI.getType(MI.getOperand(1).getReg());
542 unsigned PtrSize = PtrTy.getSizeInBits();
543 unsigned AS = PtrTy.getAddressSpace();
545 if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
546 AS != AMDGPUAS::PRIVATE_ADDRESS) &&
547 isScalarLoadLegal(MI)) {
548 const InstructionMapping &SSMapping = getInstructionMapping(
549 1, 1, getOperandsMapping(
550 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
551 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize)}),
553 AltMappings.push_back(&SSMapping);
556 const InstructionMapping &VVMapping = getInstructionMapping(
559 {AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
560 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize)}),
562 AltMappings.push_back(&VVMapping);
564 // It may be possible to have a vgpr = load sgpr mapping here, because
565 // the mubuf instructions support this kind of load, but probably for only
566 // gfx7 and older. However, the addressing mode matching in the instruction
567 // selector should be able to do a better job of detecting and selecting
568 // these kinds of loads from the vgpr = load vgpr mapping.
573 case TargetOpcode::G_SELECT: {
574 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
575 const InstructionMapping &SSMapping = getInstructionMapping(1, 1,
576 getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
577 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
578 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
579 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
581 AltMappings.push_back(&SSMapping);
583 const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
584 getOperandsMapping({AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
585 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
586 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
587 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}),
589 AltMappings.push_back(&VVMapping);
593 case TargetOpcode::G_SMIN:
594 case TargetOpcode::G_SMAX:
595 case TargetOpcode::G_UMIN:
596 case TargetOpcode::G_UMAX: {
597 static const OpRegBankEntry<3> Table[2] = {
598 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 },
600 // Scalar requires cmp+select, and extends if 16-bit.
601 // FIXME: Should there be separate costs for 32 and 16-bit
602 { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 3 }
605 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 1, 2 } };
606 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table));
608 case TargetOpcode::G_UADDE:
609 case TargetOpcode::G_USUBE:
610 case TargetOpcode::G_SADDE:
611 case TargetOpcode::G_SSUBE: {
612 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
613 const InstructionMapping &SSMapping = getInstructionMapping(1, 1,
615 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
616 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
617 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
618 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
619 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1)}),
621 AltMappings.push_back(&SSMapping);
623 const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
624 getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
625 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
626 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
627 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
628 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1)}),
630 AltMappings.push_back(&VVMapping);
633 case AMDGPU::G_BRCOND: {
634 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
636 // TODO: Change type to 32 for scalar
637 const InstructionMapping &SMapping = getInstructionMapping(
638 1, 1, getOperandsMapping(
639 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), nullptr}),
641 AltMappings.push_back(&SMapping);
643 const InstructionMapping &VMapping = getInstructionMapping(
644 1, 1, getOperandsMapping(
645 {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), nullptr }),
647 AltMappings.push_back(&VMapping);
650 case AMDGPU::G_INTRINSIC:
651 return getInstrAlternativeMappingsIntrinsic(MI, MRI);
652 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
653 return getInstrAlternativeMappingsIntrinsicWSideEffects(MI, MRI);
657 return RegisterBankInfo::getInstrAlternativeMappings(MI);
660 void AMDGPURegisterBankInfo::split64BitValueForMapping(
662 SmallVector<Register, 2> &Regs,
664 Register Reg) const {
665 assert(HalfTy.getSizeInBits() == 32);
666 MachineRegisterInfo *MRI = B.getMRI();
667 Register LoLHS = MRI->createGenericVirtualRegister(HalfTy);
668 Register HiLHS = MRI->createGenericVirtualRegister(HalfTy);
669 const RegisterBank *Bank = getRegBank(Reg, *MRI, *TRI);
670 MRI->setRegBank(LoLHS, *Bank);
671 MRI->setRegBank(HiLHS, *Bank);
673 Regs.push_back(LoLHS);
674 Regs.push_back(HiLHS);
676 B.buildInstr(AMDGPU::G_UNMERGE_VALUES)
682 /// Replace the current type each register in \p Regs has with \p NewTy
683 static void setRegsToType(MachineRegisterInfo &MRI, ArrayRef<Register> Regs,
685 for (Register Reg : Regs) {
686 assert(MRI.getType(Reg).getSizeInBits() == NewTy.getSizeInBits());
687 MRI.setType(Reg, NewTy);
691 static LLT getHalfSizedType(LLT Ty) {
693 assert(Ty.getNumElements() % 2 == 0);
694 return LLT::scalarOrVector(Ty.getNumElements() / 2, Ty.getElementType());
697 assert(Ty.getSizeInBits() % 2 == 0);
698 return LLT::scalar(Ty.getSizeInBits() / 2);
701 /// Legalize instruction \p MI where operands in \p OpIndices must be SGPRs. If
702 /// any of the required SGPR operands are VGPRs, perform a waterfall loop to
703 /// execute the instruction for each unique combination of values in all lanes
704 /// in the wave. The block will be split such that rest of the instructions are
705 /// moved to a new block.
707 /// Essentially performs this loop:
709 /// Save Execution Mask
710 /// For (Lane : Wavefront) {
711 /// Enable Lane, Disable all other lanes
712 /// SGPR = read SGPR value for current lane from VGPR
713 /// VGPRResult[Lane] = use_op SGPR
715 /// Restore Execution Mask
717 /// There is additional complexity to try for compare values to identify the
718 /// unique values used.
719 bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
721 iterator_range<MachineBasicBlock::iterator> Range,
722 SmallSet<Register, 4> &SGPROperandRegs,
723 MachineRegisterInfo &MRI) const {
724 SmallVector<Register, 4> ResultRegs;
725 SmallVector<Register, 4> InitResultRegs;
726 SmallVector<Register, 4> PhiRegs;
728 // Track use registers which have already been expanded with a readfirstlane
729 // sequence. This may have multiple uses if moving a sequence.
730 DenseMap<Register, Register> WaterfalledRegMap;
732 MachineBasicBlock &MBB = B.getMBB();
733 MachineFunction *MF = &B.getMF();
735 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
736 const unsigned WaveAndOpc = Subtarget.isWave32() ?
737 AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
738 const unsigned MovTermOpc = Subtarget.isWave32() ?
739 AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term;
740 const unsigned XorTermOpc = Subtarget.isWave32() ?
741 AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
742 const unsigned AndSaveExecOpc = Subtarget.isWave32() ?
743 AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
744 const unsigned ExecReg = Subtarget.isWave32() ?
745 AMDGPU::EXEC_LO : AMDGPU::EXEC;
748 const int OrigRangeSize = std::distance(Range.begin(), Range.end());
751 for (MachineInstr &MI : Range) {
752 for (MachineOperand &Def : MI.defs()) {
753 LLT ResTy = MRI.getType(Def.getReg());
754 const RegisterBank *DefBank = getRegBank(Def.getReg(), MRI, *TRI);
755 ResultRegs.push_back(Def.getReg());
756 Register InitReg = B.buildUndef(ResTy).getReg(0);
757 Register PhiReg = MRI.createGenericVirtualRegister(ResTy);
758 InitResultRegs.push_back(InitReg);
759 PhiRegs.push_back(PhiReg);
760 MRI.setRegBank(PhiReg, *DefBank);
761 MRI.setRegBank(InitReg, *DefBank);
765 Register SaveExecReg = MRI.createVirtualRegister(WaveRC);
766 Register InitSaveExecReg = MRI.createVirtualRegister(WaveRC);
768 // Don't bother using generic instructions/registers for the exec mask.
769 B.buildInstr(TargetOpcode::IMPLICIT_DEF)
770 .addDef(InitSaveExecReg);
772 Register PhiExec = MRI.createVirtualRegister(WaveRC);
773 Register NewExec = MRI.createVirtualRegister(WaveRC);
775 // To insert the loop we need to split the block. Move everything before this
776 // point to a new block, and insert a new empty block before this instruction.
777 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
778 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
779 MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock();
780 MachineFunction::iterator MBBI(MBB);
782 MF->insert(MBBI, LoopBB);
783 MF->insert(MBBI, RestoreExecBB);
784 MF->insert(MBBI, RemainderBB);
786 LoopBB->addSuccessor(RestoreExecBB);
787 LoopBB->addSuccessor(LoopBB);
789 // Move the rest of the block into a new block.
790 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
791 RemainderBB->splice(RemainderBB->begin(), &MBB, Range.end(), MBB.end());
793 MBB.addSuccessor(LoopBB);
794 RestoreExecBB->addSuccessor(RemainderBB);
796 B.setInsertPt(*LoopBB, LoopBB->end());
798 B.buildInstr(TargetOpcode::PHI)
800 .addReg(InitSaveExecReg)
805 for (auto Result : zip(InitResultRegs, ResultRegs, PhiRegs)) {
806 B.buildInstr(TargetOpcode::G_PHI)
807 .addDef(std::get<2>(Result))
808 .addReg(std::get<0>(Result)) // Initial value / implicit_def
810 .addReg(std::get<1>(Result)) // Mid-loop value.
814 const DebugLoc &DL = B.getDL();
816 MachineInstr &FirstInst = *Range.begin();
818 // Move the instruction into the loop. Note we moved everything after
819 // Range.end() already into a new block, so Range.end() is no longer valid.
820 LoopBB->splice(LoopBB->end(), &MBB, Range.begin(), MBB.end());
822 // Figure out the iterator range after splicing the instructions.
823 MachineBasicBlock::iterator NewBegin = FirstInst.getIterator();
824 auto NewEnd = LoopBB->end();
826 MachineBasicBlock::iterator I = Range.begin();
827 B.setInsertPt(*LoopBB, I);
831 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
833 for (MachineInstr &MI : make_range(NewBegin, NewEnd)) {
834 for (MachineOperand &Op : MI.uses()) {
835 if (!Op.isReg() || Op.isDef())
838 Register OldReg = Op.getReg();
839 if (!SGPROperandRegs.count(OldReg))
842 // See if we already processed this register in another instruction in the
844 auto OldVal = WaterfalledRegMap.find(OldReg);
845 if (OldVal != WaterfalledRegMap.end()) {
846 Op.setReg(OldVal->second);
850 LLT OpTy = MRI.getType(Op.getReg());
851 unsigned OpSize = OpTy.getSizeInBits();
853 // Can only do a readlane of 32-bit pieces.
855 // Avoid extra copies in the simple case of one 32-bit register.
856 Register CurrentLaneOpReg
857 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
858 MRI.setType(CurrentLaneOpReg, OpTy);
860 constrainGenericRegister(Op.getReg(), AMDGPU::VGPR_32RegClass, MRI);
861 // Read the next variant <- also loop target.
862 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
864 .addReg(Op.getReg());
866 Register NewCondReg = MRI.createVirtualRegister(WaveRC);
867 bool First = CondReg == AMDGPU::NoRegister;
869 CondReg = NewCondReg;
871 // Compare the just read M0 value to all possible Idx values.
872 B.buildInstr(AMDGPU::V_CMP_EQ_U32_e64)
874 .addReg(CurrentLaneOpReg)
875 .addReg(Op.getReg());
876 Op.setReg(CurrentLaneOpReg);
879 Register AndReg = MRI.createVirtualRegister(WaveRC);
881 // If there are multiple operands to consider, and the conditions.
882 B.buildInstr(WaveAndOpc)
889 LLT S32 = LLT::scalar(32);
890 SmallVector<Register, 8> ReadlanePieces;
892 // The compares can be done as 64-bit, but the extract needs to be done
895 bool Is64 = OpSize % 64 == 0;
897 LLT UnmergeTy = OpSize % 64 == 0 ? LLT::scalar(64) : LLT::scalar(32);
898 unsigned CmpOp = OpSize % 64 == 0 ? AMDGPU::V_CMP_EQ_U64_e64
899 : AMDGPU::V_CMP_EQ_U32_e64;
901 // The compares can be done as 64-bit, but the extract needs to be done
904 // Insert the unmerge before the loop.
907 auto Unmerge = B.buildUnmerge(UnmergeTy, Op.getReg());
910 unsigned NumPieces = Unmerge->getNumOperands() - 1;
911 for (unsigned PieceIdx = 0; PieceIdx != NumPieces; ++PieceIdx) {
912 Register UnmergePiece = Unmerge.getReg(PieceIdx);
914 Register CurrentLaneOpReg;
916 Register CurrentLaneOpRegLo = MRI.createGenericVirtualRegister(S32);
917 Register CurrentLaneOpRegHi = MRI.createGenericVirtualRegister(S32);
919 MRI.setRegClass(UnmergePiece, &AMDGPU::VReg_64RegClass);
920 MRI.setRegClass(CurrentLaneOpRegLo, &AMDGPU::SReg_32_XM0RegClass);
921 MRI.setRegClass(CurrentLaneOpRegHi, &AMDGPU::SReg_32_XM0RegClass);
923 // Read the next variant <- also loop target.
924 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
926 .addReg(UnmergePiece, 0, AMDGPU::sub0);
928 // Read the next variant <- also loop target.
929 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
931 .addReg(UnmergePiece, 0, AMDGPU::sub1);
934 B.buildMerge(LLT::scalar(64),
935 {CurrentLaneOpRegLo, CurrentLaneOpRegHi})
938 MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_64_XEXECRegClass);
940 if (OpTy.getScalarSizeInBits() == 64) {
941 // If we need to produce a 64-bit element vector, so use the
943 ReadlanePieces.push_back(CurrentLaneOpReg);
945 // 32-bit element type.
946 ReadlanePieces.push_back(CurrentLaneOpRegLo);
947 ReadlanePieces.push_back(CurrentLaneOpRegHi);
950 CurrentLaneOpReg = MRI.createGenericVirtualRegister(S32);
951 MRI.setRegClass(UnmergePiece, &AMDGPU::VGPR_32RegClass);
952 MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_32_XM0RegClass);
954 // Read the next variant <- also loop target.
955 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
957 .addReg(UnmergePiece);
958 ReadlanePieces.push_back(CurrentLaneOpReg);
961 Register NewCondReg = MRI.createVirtualRegister(WaveRC);
962 bool First = CondReg == AMDGPU::NoRegister;
964 CondReg = NewCondReg;
968 .addReg(CurrentLaneOpReg)
969 .addReg(UnmergePiece);
972 Register AndReg = MRI.createVirtualRegister(WaveRC);
974 // If there are multiple operands to consider, and the conditions.
975 B.buildInstr(WaveAndOpc)
983 // FIXME: Build merge seems to switch to CONCAT_VECTORS but not
985 if (OpTy.isVector()) {
986 auto Merge = B.buildBuildVector(OpTy, ReadlanePieces);
987 Op.setReg(Merge.getReg(0));
989 auto Merge = B.buildMerge(OpTy, ReadlanePieces);
990 Op.setReg(Merge.getReg(0));
993 MRI.setRegBank(Op.getReg(), AMDGPU::SGPRRegBank);
996 // Make sure we don't re-process this register again.
997 WaterfalledRegMap.insert(std::make_pair(OldReg, Op.getReg()));
1001 B.setInsertPt(*LoopBB, LoopBB->end());
1003 // Update EXEC, save the original EXEC value to VCC.
1004 B.buildInstr(AndSaveExecOpc)
1006 .addReg(CondReg, RegState::Kill);
1008 MRI.setSimpleHint(NewExec, CondReg);
1010 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
1011 B.buildInstr(XorTermOpc)
1016 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
1019 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
1020 B.buildInstr(AMDGPU::S_CBRANCH_EXECNZ)
1023 // Save the EXEC mask before the loop.
1024 BuildMI(MBB, MBB.end(), DL, TII->get(MovTermOpc), SaveExecReg)
1027 // Restore the EXEC mask after the loop.
1028 B.setMBB(*RestoreExecBB);
1029 B.buildInstr(MovTermOpc)
1031 .addReg(SaveExecReg);
1033 // Set the insert point after the original instruction, so any new
1034 // instructions will be in the remainder.
1035 B.setInsertPt(*RemainderBB, RemainderBB->begin());
1040 // Return any unique registers used by \p MI at \p OpIndices that need to be
1041 // handled in a waterfall loop. Returns these registers in \p
1042 // SGPROperandRegs. Returns true if there are any operansd to handle and a
1043 // waterfall loop is necessary.
1044 bool AMDGPURegisterBankInfo::collectWaterfallOperands(
1045 SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI,
1046 MachineRegisterInfo &MRI, ArrayRef<unsigned> OpIndices) const {
1047 for (unsigned Op : OpIndices) {
1048 assert(MI.getOperand(Op).isUse());
1049 Register Reg = MI.getOperand(Op).getReg();
1050 const RegisterBank *OpBank = getRegBank(Reg, MRI, *TRI);
1051 if (OpBank->getID() == AMDGPU::VGPRRegBankID)
1052 SGPROperandRegs.insert(Reg);
1055 // No operands need to be replaced, so no need to loop.
1056 return !SGPROperandRegs.empty();
1059 bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
1060 MachineIRBuilder &B, MachineInstr &MI, MachineRegisterInfo &MRI,
1061 ArrayRef<unsigned> OpIndices) const {
1062 // Use a set to avoid extra readfirstlanes in the case where multiple operands
1063 // are the same register.
1064 SmallSet<Register, 4> SGPROperandRegs;
1066 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI, OpIndices))
1069 MachineBasicBlock::iterator I = MI.getIterator();
1070 return executeInWaterfallLoop(B, make_range(I, std::next(I)),
1071 SGPROperandRegs, MRI);
1074 bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
1075 MachineInstr &MI, MachineRegisterInfo &MRI,
1076 ArrayRef<unsigned> OpIndices) const {
1077 MachineIRBuilder B(MI);
1078 return executeInWaterfallLoop(B, MI, MRI, OpIndices);
1081 // Legalize an operand that must be an SGPR by inserting a readfirstlane.
1082 void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane(
1083 MachineInstr &MI, MachineRegisterInfo &MRI, unsigned OpIdx) const {
1084 Register Reg = MI.getOperand(OpIdx).getReg();
1085 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
1086 if (Bank != &AMDGPU::VGPRRegBank)
1089 MachineIRBuilder B(MI);
1090 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
1091 B.buildInstr(AMDGPU::V_READFIRSTLANE_B32)
1095 MRI.setType(SGPR, MRI.getType(Reg));
1097 const TargetRegisterClass *Constrained =
1098 constrainGenericRegister(Reg, AMDGPU::VGPR_32RegClass, MRI);
1100 assert(Constrained && "Failed to constrain readfirstlane src reg");
1102 MI.getOperand(OpIdx).setReg(SGPR);
1105 /// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
1106 /// rest will be in the remainder.
1107 static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) {
1108 unsigned TotalSize = Ty.getSizeInBits();
1110 return {LLT::scalar(FirstSize), LLT::scalar(TotalSize - FirstSize)};
1112 LLT EltTy = Ty.getElementType();
1113 unsigned EltSize = EltTy.getSizeInBits();
1114 assert(FirstSize % EltSize == 0);
1116 unsigned FirstPartNumElts = FirstSize / EltSize;
1117 unsigned RemainderElts = (TotalSize - FirstSize) / EltSize;
1119 return {LLT::scalarOrVector(FirstPartNumElts, EltTy),
1120 LLT::scalarOrVector(RemainderElts, EltTy)};
1123 static LLT widen96To128(LLT Ty) {
1125 return LLT::scalar(128);
1127 LLT EltTy = Ty.getElementType();
1128 assert(128 % EltTy.getSizeInBits() == 0);
1129 return LLT::vector(128 / EltTy.getSizeInBits(), EltTy);
1132 bool AMDGPURegisterBankInfo::applyMappingLoad(MachineInstr &MI,
1133 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1134 MachineRegisterInfo &MRI) const {
1135 Register DstReg = MI.getOperand(0).getReg();
1136 const LLT LoadTy = MRI.getType(DstReg);
1137 unsigned LoadSize = LoadTy.getSizeInBits();
1138 const unsigned MaxNonSmrdLoadSize = 128;
1140 const RegisterBank *PtrBank =
1141 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
1142 if (PtrBank == &AMDGPU::SGPRRegBank) {
1143 // If the pointer is an SGPR, we ordinarily have nothing to do.
1147 MachineMemOperand *MMO = *MI.memoperands_begin();
1148 Register PtrReg = MI.getOperand(1).getReg();
1149 // 96-bit loads are only available for vector loads. We need to split this
1150 // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
1152 MachineIRBuilder B(MI);
1153 ApplyRegBankMapping O(*this, MRI, &AMDGPU::SGPRRegBank);
1154 GISelObserverWrapper Observer(&O);
1155 B.setChangeObserver(Observer);
1157 if (MMO->getAlign() < Align(16)) {
1159 std::tie(Part64, Part32) = splitUnequalType(LoadTy, 64);
1160 auto Load0 = B.buildLoadFromOffset(Part64, PtrReg, *MMO, 0);
1161 auto Load1 = B.buildLoadFromOffset(Part32, PtrReg, *MMO, 8);
1163 auto Undef = B.buildUndef(LoadTy);
1164 auto Ins0 = B.buildInsert(LoadTy, Undef, Load0, 0);
1165 B.buildInsert(MI.getOperand(0), Ins0, Load1, 64);
1167 LLT WiderTy = widen96To128(LoadTy);
1168 auto WideLoad = B.buildLoadFromOffset(WiderTy, PtrReg, *MMO, 0);
1169 B.buildExtract(MI.getOperand(0), WideLoad, 0);
1172 MI.eraseFromParent();
1176 // 128-bit loads are supported for all instruction types.
1177 if (LoadSize <= MaxNonSmrdLoadSize)
1180 SmallVector<Register, 16> DefRegs(OpdMapper.getVRegs(0));
1181 SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(1));
1183 if (SrcRegs.empty())
1184 SrcRegs.push_back(MI.getOperand(1).getReg());
1186 assert(LoadSize % MaxNonSmrdLoadSize == 0);
1188 // RegBankSelect only emits scalar types, so we need to reset the pointer
1189 // operand to a pointer type.
1190 Register BasePtrReg = SrcRegs[0];
1191 LLT PtrTy = MRI.getType(MI.getOperand(1).getReg());
1192 MRI.setType(BasePtrReg, PtrTy);
1194 MachineIRBuilder B(MI);
1196 unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize;
1197 const LLT LoadSplitTy = LoadTy.divide(NumSplitParts);
1198 ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank);
1199 GISelObserverWrapper Observer(&O);
1200 B.setChangeObserver(Observer);
1201 LegalizerHelper Helper(B.getMF(), Observer, B);
1203 if (LoadTy.isVector()) {
1204 if (Helper.fewerElementsVector(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized)
1207 if (Helper.narrowScalar(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized)
1211 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
1215 bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
1217 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1218 MachineRegisterInfo &MRI) const {
1219 const MachineFunction &MF = *MI.getMF();
1220 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1221 const auto &TFI = *ST.getFrameLowering();
1223 // Guard in case the stack growth direction ever changes with scratch
1225 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown)
1228 Register Dst = MI.getOperand(0).getReg();
1229 Register AllocSize = MI.getOperand(1).getReg();
1230 Align Alignment = assumeAligned(MI.getOperand(2).getImm());
1232 const RegisterBank *SizeBank = getRegBank(AllocSize, MRI, *TRI);
1234 // TODO: Need to emit a wave reduction to get the maximum size.
1235 if (SizeBank != &AMDGPU::SGPRRegBank)
1238 LLT PtrTy = MRI.getType(Dst);
1239 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
1241 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1242 Register SPReg = Info->getStackPtrOffsetReg();
1243 ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank);
1244 GISelObserverWrapper Observer(&ApplyBank);
1246 MachineIRBuilder B(MI);
1247 B.setChangeObserver(Observer);
1249 auto WaveSize = B.buildConstant(LLT::scalar(32), ST.getWavefrontSizeLog2());
1250 auto ScaledSize = B.buildShl(IntPtrTy, AllocSize, WaveSize);
1252 auto SPCopy = B.buildCopy(PtrTy, SPReg);
1253 if (Alignment > TFI.getStackAlign()) {
1254 auto PtrAdd = B.buildPtrAdd(PtrTy, SPCopy, ScaledSize);
1255 B.buildMaskLowPtrBits(Dst, PtrAdd,
1256 Log2(Alignment) + ST.getWavefrontSizeLog2());
1258 B.buildPtrAdd(Dst, SPCopy, ScaledSize);
1261 MI.eraseFromParent();
1265 bool AMDGPURegisterBankInfo::applyMappingImage(
1266 MachineInstr &MI, const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1267 MachineRegisterInfo &MRI, int RsrcIdx) const {
1268 const int NumDefs = MI.getNumExplicitDefs();
1270 // The reported argument index is relative to the IR intrinsic call arguments,
1271 // so we need to shift by the number of defs and the intrinsic ID.
1272 RsrcIdx += NumDefs + 1;
1274 // Insert copies to VGPR arguments.
1275 applyDefaultMapping(OpdMapper);
1277 // Fixup any SGPR arguments.
1278 SmallVector<unsigned, 4> SGPRIndexes;
1279 for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) {
1280 if (!MI.getOperand(I).isReg())
1283 // If this intrinsic has a sampler, it immediately follows rsrc.
1284 if (I == RsrcIdx || I == RsrcIdx + 1)
1285 SGPRIndexes.push_back(I);
1288 executeInWaterfallLoop(MI, MRI, SGPRIndexes);
1292 static Register getSrcRegIgnoringCopies(const MachineRegisterInfo &MRI,
1294 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1298 // TODO: Guard against this being an implicit def
1299 return Def->getOperand(0).getReg();
1302 // Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
1303 // the three offsets (voffset, soffset and instoffset)
1304 static unsigned setBufferOffsets(MachineIRBuilder &B,
1305 const AMDGPURegisterBankInfo &RBI,
1306 Register CombinedOffset, Register &VOffsetReg,
1307 Register &SOffsetReg, int64_t &InstOffsetVal,
1309 const LLT S32 = LLT::scalar(32);
1310 MachineRegisterInfo *MRI = B.getMRI();
1312 if (Optional<int64_t> Imm = getConstantVRegVal(CombinedOffset, *MRI)) {
1313 uint32_t SOffset, ImmOffset;
1314 if (AMDGPU::splitMUBUFOffset(*Imm, SOffset, ImmOffset, &RBI.Subtarget,
1316 VOffsetReg = B.buildConstant(S32, 0).getReg(0);
1317 SOffsetReg = B.buildConstant(S32, SOffset).getReg(0);
1318 InstOffsetVal = ImmOffset;
1320 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1321 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1322 return SOffset + ImmOffset;
1328 MachineInstr *Unused;
1330 std::tie(Base, Offset, Unused)
1331 = AMDGPU::getBaseWithConstantOffset(*MRI, CombinedOffset);
1333 uint32_t SOffset, ImmOffset;
1334 if (Offset > 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset,
1335 &RBI.Subtarget, Alignment)) {
1336 if (RBI.getRegBank(Base, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) {
1338 SOffsetReg = B.buildConstant(S32, SOffset).getReg(0);
1339 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1340 InstOffsetVal = ImmOffset;
1341 return 0; // XXX - Why is this 0?
1344 // If we have SGPR base, we can use it for soffset.
1346 VOffsetReg = B.buildConstant(S32, 0).getReg(0);
1347 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1349 InstOffsetVal = ImmOffset;
1350 return 0; // XXX - Why is this 0?
1354 // Handle the variable sgpr + vgpr case.
1355 if (MachineInstr *Add = getOpcodeDef(AMDGPU::G_ADD, CombinedOffset, *MRI)) {
1356 Register Src0 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(1).getReg());
1357 Register Src1 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(2).getReg());
1359 const RegisterBank *Src0Bank = RBI.getRegBank(Src0, *MRI, *RBI.TRI);
1360 const RegisterBank *Src1Bank = RBI.getRegBank(Src1, *MRI, *RBI.TRI);
1362 if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) {
1368 if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) {
1375 // Ensure we have a VGPR for the combined offset. This could be an issue if we
1376 // have an SGPR offset and a VGPR resource.
1377 if (RBI.getRegBank(CombinedOffset, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) {
1378 VOffsetReg = CombinedOffset;
1380 VOffsetReg = B.buildCopy(S32, CombinedOffset).getReg(0);
1381 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1384 SOffsetReg = B.buildConstant(S32, 0).getReg(0);
1385 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1389 bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
1390 const OperandsMapper &OpdMapper) const {
1391 MachineInstr &MI = OpdMapper.getMI();
1392 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1394 const LLT S32 = LLT::scalar(32);
1395 Register Dst = MI.getOperand(0).getReg();
1396 LLT Ty = MRI.getType(Dst);
1398 const RegisterBank *RSrcBank =
1399 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
1400 const RegisterBank *OffsetBank =
1401 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
1402 if (RSrcBank == &AMDGPU::SGPRRegBank &&
1403 OffsetBank == &AMDGPU::SGPRRegBank)
1404 return true; // Legal mapping
1406 // FIXME: 96-bit case was widened during legalize. We neeed to narrow it back
1407 // here but don't have an MMO.
1409 unsigned LoadSize = Ty.getSizeInBits();
1411 if (LoadSize == 256 || LoadSize == 512) {
1412 NumLoads = LoadSize / 128;
1413 Ty = Ty.divide(NumLoads);
1416 // Use the alignment to ensure that the required offsets will fit into the
1417 // immediate offsets.
1418 const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
1420 MachineIRBuilder B(MI);
1421 MachineFunction &MF = B.getMF();
1425 int64_t ImmOffset = 0;
1427 unsigned MMOOffset = setBufferOffsets(B, *this, MI.getOperand(2).getReg(),
1428 VOffset, SOffset, ImmOffset, Alignment);
1430 // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
1431 // can, but we neeed to track an MMO for that.
1432 const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
1433 const Align MemAlign(4); // FIXME: ABI type alignment?
1434 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
1435 MachinePointerInfo(),
1436 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1437 MachineMemOperand::MOInvariant,
1440 BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset, MemSize);
1442 // If only the offset is divergent, emit a MUBUF buffer load instead. We can
1443 // assume that the buffer is unswizzled.
1445 Register RSrc = MI.getOperand(1).getReg();
1446 Register VIndex = B.buildConstant(S32, 0).getReg(0);
1447 B.getMRI()->setRegBank(VIndex, AMDGPU::VGPRRegBank);
1449 SmallVector<Register, 4> LoadParts(NumLoads);
1451 MachineBasicBlock::iterator MII = MI.getIterator();
1452 MachineInstrSpan Span(MII, &B.getMBB());
1454 for (int i = 0; i < NumLoads; ++i) {
1455 if (NumLoads == 1) {
1458 LoadParts[i] = MRI.createGenericVirtualRegister(Ty);
1459 MRI.setRegBank(LoadParts[i], AMDGPU::VGPRRegBank);
1462 MachineMemOperand *MMO = BaseMMO;
1464 BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset + 16 * i, MemSize);
1466 B.buildInstr(AMDGPU::G_AMDGPU_BUFFER_LOAD)
1467 .addDef(LoadParts[i]) // vdata
1468 .addUse(RSrc) // rsrc
1469 .addUse(VIndex) // vindex
1470 .addUse(VOffset) // voffset
1471 .addUse(SOffset) // soffset
1472 .addImm(ImmOffset + 16 * i) // offset(imm)
1473 .addImm(0) // cachepolicy, swizzled buffer(imm)
1474 .addImm(0) // idxen(imm)
1475 .addMemOperand(MMO);
1478 // TODO: If only the resource is a VGPR, it may be better to execute the
1479 // scalar load in the waterfall loop if the resource is expected to frequently
1480 // be dynamically uniform.
1481 if (RSrcBank != &AMDGPU::SGPRRegBank) {
1482 // Remove the original instruction to avoid potentially confusing the
1483 // waterfall loop logic.
1484 B.setInstr(*Span.begin());
1485 MI.eraseFromParent();
1487 SmallSet<Register, 4> OpsToWaterfall;
1489 OpsToWaterfall.insert(RSrc);
1490 executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()),
1491 OpsToWaterfall, MRI);
1494 if (NumLoads != 1) {
1496 B.buildConcatVectors(Dst, LoadParts);
1498 B.buildMerge(Dst, LoadParts);
1501 // We removed the instruction earlier with a waterfall loop.
1502 if (RSrcBank == &AMDGPU::SGPRRegBank)
1503 MI.eraseFromParent();
1508 bool AMDGPURegisterBankInfo::applyMappingBFEIntrinsic(
1509 const OperandsMapper &OpdMapper, bool Signed) const {
1510 MachineInstr &MI = OpdMapper.getMI();
1511 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1513 // Insert basic copies
1514 applyDefaultMapping(OpdMapper);
1516 Register DstReg = MI.getOperand(0).getReg();
1517 LLT Ty = MRI.getType(DstReg);
1519 const LLT S32 = LLT::scalar(32);
1521 const RegisterBank *DstBank =
1522 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
1523 if (DstBank == &AMDGPU::VGPRRegBank) {
1527 // TODO: 64-bit version is scalar only, so we need to expand this.
1531 Register SrcReg = MI.getOperand(2).getReg();
1532 Register OffsetReg = MI.getOperand(3).getReg();
1533 Register WidthReg = MI.getOperand(4).getReg();
1535 // The scalar form packs the offset and width in a single operand.
1537 ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank);
1538 GISelObserverWrapper Observer(&ApplyBank);
1539 MachineIRBuilder B(MI);
1540 B.setChangeObserver(Observer);
1542 // Ensure the high bits are clear to insert the offset.
1543 auto OffsetMask = B.buildConstant(S32, maskTrailingOnes<unsigned>(6));
1544 auto ClampOffset = B.buildAnd(S32, OffsetReg, OffsetMask);
1546 // Zeros out the low bits, so don't bother clamping the input value.
1547 auto ShiftWidth = B.buildShl(S32, WidthReg, B.buildConstant(S32, 16));
1549 // Transformation function, pack the offset and width of a BFE into
1550 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1551 // source, bits [5:0] contain the offset and bits [22:16] the width.
1552 auto MergedInputs = B.buildOr(S32, ClampOffset, ShiftWidth);
1554 // TODO: It might be worth using a pseudo here to avoid scc clobber and
1555 // register class constraints.
1556 unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) :
1557 (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64);
1559 auto MIB = B.buildInstr(Opc, {DstReg}, {SrcReg, MergedInputs});
1560 if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this))
1561 llvm_unreachable("failed to constrain BFE");
1563 MI.eraseFromParent();
1567 // FIXME: Duplicated from LegalizerHelper
1568 static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
1570 case TargetOpcode::G_SMIN:
1571 return CmpInst::ICMP_SLT;
1572 case TargetOpcode::G_SMAX:
1573 return CmpInst::ICMP_SGT;
1574 case TargetOpcode::G_UMIN:
1575 return CmpInst::ICMP_ULT;
1576 case TargetOpcode::G_UMAX:
1577 return CmpInst::ICMP_UGT;
1579 llvm_unreachable("not in integer min/max");
1583 static unsigned minMaxToExtend(unsigned Opc) {
1585 case TargetOpcode::G_SMIN:
1586 case TargetOpcode::G_SMAX:
1587 return TargetOpcode::G_SEXT;
1588 case TargetOpcode::G_UMIN:
1589 case TargetOpcode::G_UMAX:
1590 return TargetOpcode::G_ZEXT;
1592 llvm_unreachable("not in integer min/max");
1596 // Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding
1597 // any illegal vector extend or unmerge operations.
1598 static std::pair<Register, Register>
1599 unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) {
1600 const LLT S32 = LLT::scalar(32);
1601 auto Bitcast = B.buildBitcast(S32, Src);
1603 if (ExtOpcode == TargetOpcode::G_SEXT) {
1604 auto ExtLo = B.buildSExtInReg(S32, Bitcast, 16);
1605 auto ShiftHi = B.buildAShr(S32, Bitcast, B.buildConstant(S32, 16));
1606 return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0));
1609 auto ShiftHi = B.buildLShr(S32, Bitcast, B.buildConstant(S32, 16));
1610 if (ExtOpcode == TargetOpcode::G_ZEXT) {
1611 auto ExtLo = B.buildAnd(S32, Bitcast, B.buildConstant(S32, 0xffff));
1612 return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0));
1615 assert(ExtOpcode == TargetOpcode::G_ANYEXT);
1616 return std::make_pair(Bitcast.getReg(0), ShiftHi.getReg(0));
1619 static MachineInstr *buildExpandedScalarMinMax(MachineIRBuilder &B,
1620 CmpInst::Predicate Pred,
1621 Register Dst, Register Src0,
1623 const LLT CmpType = LLT::scalar(32);
1624 auto Cmp = B.buildICmp(Pred, CmpType, Src0, Src1);
1625 return B.buildSelect(Dst, Cmp, Src0, Src1);
1628 // FIXME: Duplicated from LegalizerHelper, except changing the boolean type.
1629 void AMDGPURegisterBankInfo::lowerScalarMinMax(MachineIRBuilder &B,
1630 MachineInstr &MI) const {
1631 Register Dst = MI.getOperand(0).getReg();
1632 Register Src0 = MI.getOperand(1).getReg();
1633 Register Src1 = MI.getOperand(2).getReg();
1635 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
1636 MachineInstr *Sel = buildExpandedScalarMinMax(B, Pred, Dst, Src0, Src1);
1638 Register CmpReg = Sel->getOperand(1).getReg();
1639 B.getMRI()->setRegBank(CmpReg, AMDGPU::SGPRRegBank);
1640 MI.eraseFromParent();
1643 // For cases where only a single copy is inserted for matching register banks.
1644 // Replace the register in the instruction operand
1645 static bool substituteSimpleCopyRegs(
1646 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) {
1647 SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx));
1648 if (!SrcReg.empty()) {
1649 assert(SrcReg.size() == 1);
1650 OpdMapper.getMI().getOperand(OpIdx).setReg(SrcReg[0]);
1657 /// Handle register layout difference for f16 images for some subtargets.
1658 Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
1659 MachineRegisterInfo &MRI,
1660 Register Reg) const {
1661 if (!Subtarget.hasUnpackedD16VMem())
1664 const LLT S16 = LLT::scalar(16);
1665 LLT StoreVT = MRI.getType(Reg);
1666 if (!StoreVT.isVector() || StoreVT.getElementType() != S16)
1669 auto Unmerge = B.buildUnmerge(S16, Reg);
1672 SmallVector<Register, 4> WideRegs;
1673 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
1674 WideRegs.push_back(Unmerge.getReg(I));
1676 const LLT S32 = LLT::scalar(32);
1677 int NumElts = StoreVT.getNumElements();
1679 return B.buildMerge(LLT::vector(NumElts, S32), WideRegs).getReg(0);
1682 static std::pair<Register, unsigned>
1683 getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
1685 if (mi_match(Reg, MRI, m_ICst(Const)))
1686 return std::make_pair(Register(), Const);
1689 if (mi_match(Reg, MRI, m_GAdd(m_Reg(Base), m_ICst(Const))))
1690 return std::make_pair(Base, Const);
1692 // TODO: Handle G_OR used for add case
1693 return std::make_pair(Reg, 0);
1696 std::pair<Register, unsigned>
1697 AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
1698 Register OrigOffset) const {
1699 const unsigned MaxImm = 4095;
1702 const LLT S32 = LLT::scalar(32);
1704 std::tie(BaseReg, ImmOffset) = getBaseWithConstantOffset(*B.getMRI(),
1708 if (ImmOffset != 0) {
1709 // If the immediate value is too big for the immoffset field, put the value
1710 // and -4096 into the immoffset field so that the value that is copied/added
1711 // for the voffset field is a multiple of 4096, and it stands more chance
1712 // of being CSEd with the copy/add for another similar load/store.
1713 // However, do not do that rounding down to a multiple of 4096 if that is a
1714 // negative number, as it appears to be illegal to have a negative offset
1715 // in the vgpr, even if adding the immediate offset makes it positive.
1716 unsigned Overflow = ImmOffset & ~MaxImm;
1717 ImmOffset -= Overflow;
1718 if ((int32_t)Overflow < 0) {
1719 Overflow += ImmOffset;
1724 if (Overflow != 0) {
1726 BaseReg = B.buildConstant(S32, Overflow).getReg(0);
1728 auto OverflowVal = B.buildConstant(S32, Overflow);
1729 BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
1735 BaseReg = B.buildConstant(S32, 0).getReg(0);
1737 return {BaseReg, C1};
1740 static bool isZero(Register Reg, MachineRegisterInfo &MRI) {
1742 return mi_match(Reg, MRI, m_ICst(C)) && C == 0;
1745 static unsigned extractGLC(unsigned CachePolicy) {
1746 return CachePolicy & 1;
1749 static unsigned extractSLC(unsigned CachePolicy) {
1750 return (CachePolicy >> 1) & 1;
1753 static unsigned extractDLC(unsigned CachePolicy) {
1754 return (CachePolicy >> 2) & 1;
1758 AMDGPURegisterBankInfo::selectStoreIntrinsic(MachineIRBuilder &B,
1759 MachineInstr &MI) const {
1760 MachineRegisterInfo &MRI = *B.getMRI();
1761 executeInWaterfallLoop(B, MI, MRI, {2, 4});
1763 // FIXME: DAG lowering brokenly changes opcode based on FP vs. integer.
1765 Register VData = MI.getOperand(1).getReg();
1766 LLT Ty = MRI.getType(VData);
1768 int EltSize = Ty.getScalarSizeInBits();
1769 int Size = Ty.getSizeInBits();
1771 // FIXME: Broken integer truncstore.
1773 report_fatal_error("unhandled intrinsic store");
1775 // FIXME: Verifier should enforce 1 MMO for these intrinsics.
1776 const int MemSize = (*MI.memoperands_begin())->getSize();
1779 Register RSrc = MI.getOperand(2).getReg();
1780 Register VOffset = MI.getOperand(3).getReg();
1781 Register SOffset = MI.getOperand(4).getReg();
1782 unsigned CachePolicy = MI.getOperand(5).getImm();
1785 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
1787 const bool Offen = !isZero(VOffset, MRI);
1789 unsigned Opc = AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact;
1790 switch (8 * MemSize) {
1792 Opc = Offen ? AMDGPU::BUFFER_STORE_BYTE_OFFEN_exact :
1793 AMDGPU::BUFFER_STORE_BYTE_OFFSET_exact;
1796 Opc = Offen ? AMDGPU::BUFFER_STORE_SHORT_OFFEN_exact :
1797 AMDGPU::BUFFER_STORE_SHORT_OFFSET_exact;
1800 Opc = Offen ? AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact :
1801 AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact;
1803 Opc = AMDGPU::getMUBUFOpcode(Opc, Size / 32);
1808 // Set the insertion point back to the instruction in case it was moved into a
1812 MachineInstrBuilder MIB = B.buildInstr(Opc)
1816 MIB.addUse(VOffset);
1821 .addImm(extractGLC(CachePolicy))
1822 .addImm(extractSLC(CachePolicy))
1823 .addImm(0) // tfe: FIXME: Remove from inst
1824 .addImm(extractDLC(CachePolicy))
1827 // FIXME: We need a way to report failure from applyMappingImpl.
1828 // Insert constrain copies before inserting the loop.
1829 if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this))
1830 report_fatal_error("failed to constrain selected store intrinsic");
1835 bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
1836 Register SrcReg) const {
1837 MachineRegisterInfo &MRI = *B.getMRI();
1838 LLT SrcTy = MRI.getType(SrcReg);
1839 if (SrcTy.getSizeInBits() == 32) {
1840 // Use a v_mov_b32 here to make the exec dependency explicit.
1841 B.buildInstr(AMDGPU::V_MOV_B32_e32)
1844 return constrainGenericRegister(DstReg, AMDGPU::VGPR_32RegClass, MRI) &&
1845 constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI);
1848 Register TmpReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1849 Register TmpReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1851 B.buildInstr(AMDGPU::V_MOV_B32_e32)
1853 .addUse(SrcReg, 0, AMDGPU::sub0);
1854 B.buildInstr(AMDGPU::V_MOV_B32_e32)
1856 .addUse(SrcReg, 0, AMDGPU::sub1);
1857 B.buildInstr(AMDGPU::REG_SEQUENCE)
1860 .addImm(AMDGPU::sub0)
1862 .addImm(AMDGPU::sub1);
1864 return constrainGenericRegister(SrcReg, AMDGPU::SReg_64RegClass, MRI) &&
1865 constrainGenericRegister(DstReg, AMDGPU::VReg_64RegClass, MRI);
1868 /// Utility function for pushing dynamic vector indexes with a constant offset
1869 /// into waterwall loops.
1870 static void reinsertVectorIndexAdd(MachineIRBuilder &B,
1871 MachineInstr &IdxUseInstr,
1873 unsigned ConstOffset) {
1874 MachineRegisterInfo &MRI = *B.getMRI();
1875 const LLT S32 = LLT::scalar(32);
1876 Register WaterfallIdx = IdxUseInstr.getOperand(OpIdx).getReg();
1877 B.setInsertPt(*IdxUseInstr.getParent(), IdxUseInstr.getIterator());
1879 auto MaterializedOffset = B.buildConstant(S32, ConstOffset);
1881 auto Add = B.buildAdd(S32, WaterfallIdx, MaterializedOffset);
1882 MRI.setRegBank(MaterializedOffset.getReg(0), AMDGPU::SGPRRegBank);
1883 MRI.setRegBank(Add.getReg(0), AMDGPU::SGPRRegBank);
1884 IdxUseInstr.getOperand(OpIdx).setReg(Add.getReg(0));
1887 /// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the
1888 /// original 32-bit source value (to be inserted in the low part of the combined
1889 /// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit
1891 static void extendLow32IntoHigh32(MachineIRBuilder &B,
1892 Register Hi32Reg, Register Lo32Reg,
1894 const RegisterBank &RegBank,
1895 bool IsBooleanSrc = false) {
1896 if (ExtOpc == AMDGPU::G_ZEXT) {
1897 B.buildConstant(Hi32Reg, 0);
1898 } else if (ExtOpc == AMDGPU::G_SEXT) {
1900 // If we know the original source was an s1, the high half is the same as
1902 B.buildCopy(Hi32Reg, Lo32Reg);
1904 // Replicate sign bit from 32-bit extended part.
1905 auto ShiftAmt = B.buildConstant(LLT::scalar(32), 31);
1906 B.getMRI()->setRegBank(ShiftAmt.getReg(0), RegBank);
1907 B.buildAShr(Hi32Reg, Lo32Reg, ShiftAmt);
1910 assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension");
1911 B.buildUndef(Hi32Reg);
1915 bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect(
1916 MachineInstr &MI, MachineRegisterInfo &MRI,
1917 const OperandsMapper &OpdMapper) const {
1919 Register VecReg = MI.getOperand(1).getReg();
1920 Register Idx = MI.getOperand(2).getReg();
1922 const RegisterBank &IdxBank =
1923 *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
1925 bool IsDivergentIdx = IdxBank == AMDGPU::VGPRRegBank;
1927 LLT VecTy = MRI.getType(VecReg);
1928 unsigned EltSize = VecTy.getScalarSizeInBits();
1929 unsigned NumElem = VecTy.getNumElements();
1931 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
1935 MachineIRBuilder B(MI);
1936 LLT S32 = LLT::scalar(32);
1938 const RegisterBank &DstBank =
1939 *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
1940 const RegisterBank &SrcBank =
1941 *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
1943 const RegisterBank &CCBank =
1944 (DstBank == AMDGPU::SGPRRegBank &&
1945 SrcBank == AMDGPU::SGPRRegBank &&
1946 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
1947 : AMDGPU::VCCRegBank;
1948 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1);
1950 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
1951 Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg();
1952 MRI.setRegBank(Idx, AMDGPU::VGPRRegBank);
1955 LLT EltTy = VecTy.getScalarType();
1956 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
1957 unsigned NumLanes = DstRegs.size();
1961 EltTy = MRI.getType(DstRegs[0]);
1963 auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg);
1964 SmallVector<Register, 2> Res(NumLanes);
1965 for (unsigned L = 0; L < NumLanes; ++L)
1966 Res[L] = UnmergeToEltTy.getReg(L);
1968 for (unsigned I = 1; I < NumElem; ++I) {
1969 auto IC = B.buildConstant(S32, I);
1970 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank);
1971 auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC);
1972 MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank);
1974 for (unsigned L = 0; L < NumLanes; ++L) {
1975 auto S = B.buildSelect(EltTy, Cmp,
1976 UnmergeToEltTy.getReg(I * NumLanes + L), Res[L]);
1978 for (unsigned N : { 0, 2, 3 })
1979 MRI.setRegBank(S->getOperand(N).getReg(), DstBank);
1981 Res[L] = S->getOperand(0).getReg();
1985 for (unsigned L = 0; L < NumLanes; ++L) {
1986 Register DstReg = (NumLanes == 1) ? MI.getOperand(0).getReg() : DstRegs[L];
1987 B.buildCopy(DstReg, Res[L]);
1988 MRI.setRegBank(DstReg, DstBank);
1991 MRI.setRegBank(MI.getOperand(0).getReg(), DstBank);
1992 MI.eraseFromParent();
1997 bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect(
1998 MachineInstr &MI, MachineRegisterInfo &MRI,
1999 const OperandsMapper &OpdMapper) const {
2001 Register VecReg = MI.getOperand(1).getReg();
2002 Register Idx = MI.getOperand(3).getReg();
2004 const RegisterBank &IdxBank =
2005 *OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank;
2007 bool IsDivergentIdx = IdxBank == AMDGPU::VGPRRegBank;
2009 LLT VecTy = MRI.getType(VecReg);
2010 unsigned EltSize = VecTy.getScalarSizeInBits();
2011 unsigned NumElem = VecTy.getNumElements();
2013 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
2017 MachineIRBuilder B(MI);
2018 LLT S32 = LLT::scalar(32);
2020 const RegisterBank &DstBank =
2021 *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2022 const RegisterBank &SrcBank =
2023 *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2024 const RegisterBank &InsBank =
2025 *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2027 const RegisterBank &CCBank =
2028 (DstBank == AMDGPU::SGPRRegBank &&
2029 SrcBank == AMDGPU::SGPRRegBank &&
2030 InsBank == AMDGPU::SGPRRegBank &&
2031 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
2032 : AMDGPU::VCCRegBank;
2033 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1);
2035 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
2036 Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg();
2037 MRI.setRegBank(Idx, AMDGPU::VGPRRegBank);
2040 LLT EltTy = VecTy.getScalarType();
2041 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2));
2042 unsigned NumLanes = InsRegs.size();
2045 InsRegs.push_back(MI.getOperand(2).getReg());
2047 EltTy = MRI.getType(InsRegs[0]);
2050 auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg);
2051 SmallVector<Register, 16> Ops(NumElem * NumLanes);
2053 for (unsigned I = 0; I < NumElem; ++I) {
2054 auto IC = B.buildConstant(S32, I);
2055 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank);
2056 auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC);
2057 MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank);
2059 for (unsigned L = 0; L < NumLanes; ++L) {
2060 auto S = B.buildSelect(EltTy, Cmp, InsRegs[L],
2061 UnmergeToEltTy.getReg(I * NumLanes + L));
2063 for (unsigned N : { 0, 2, 3 })
2064 MRI.setRegBank(S->getOperand(N).getReg(), DstBank);
2066 Ops[I * NumLanes + L] = S->getOperand(0).getReg();
2070 LLT MergeTy = LLT::vector(Ops.size(), EltTy);
2071 if (MergeTy == MRI.getType(MI.getOperand(0).getReg())) {
2072 B.buildBuildVector(MI.getOperand(0), Ops);
2074 auto Vec = B.buildBuildVector(MergeTy, Ops);
2075 MRI.setRegBank(Vec->getOperand(0).getReg(), DstBank);
2076 B.buildBitcast(MI.getOperand(0).getReg(), Vec);
2079 MRI.setRegBank(MI.getOperand(0).getReg(), DstBank);
2080 MI.eraseFromParent();
2085 void AMDGPURegisterBankInfo::applyMappingImpl(
2086 const OperandsMapper &OpdMapper) const {
2087 MachineInstr &MI = OpdMapper.getMI();
2088 unsigned Opc = MI.getOpcode();
2089 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2091 case AMDGPU::G_PHI: {
2092 Register DstReg = MI.getOperand(0).getReg();
2093 LLT DstTy = MRI.getType(DstReg);
2094 if (DstTy != LLT::scalar(1))
2097 const LLT S32 = LLT::scalar(32);
2098 const RegisterBank *DstBank =
2099 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2100 if (DstBank == &AMDGPU::VCCRegBank) {
2101 applyDefaultMapping(OpdMapper);
2102 // The standard handling only considers the result register bank for
2103 // phis. For VCC, blindly inserting a copy when the phi is lowered will
2104 // produce an invalid copy. We can only copy with some kind of compare to
2105 // get a vector boolean result. Insert a regitser bank copy that will be
2106 // correctly lowered to a compare.
2107 MachineIRBuilder B(*MI.getParent()->getParent());
2109 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2110 Register SrcReg = MI.getOperand(I).getReg();
2111 const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI);
2113 if (SrcBank != &AMDGPU::VCCRegBank) {
2114 MachineBasicBlock *SrcMBB = MI.getOperand(I + 1).getMBB();
2115 B.setInsertPt(*SrcMBB, SrcMBB->getFirstTerminator());
2117 auto Copy = B.buildCopy(LLT::scalar(1), SrcReg);
2118 MRI.setRegBank(Copy.getReg(0), AMDGPU::VCCRegBank);
2119 MI.getOperand(I).setReg(Copy.getReg(0));
2126 // Phi handling is strange and only considers the bank of the destination.
2127 substituteSimpleCopyRegs(OpdMapper, 0);
2129 // Promote SGPR/VGPR booleans to s32
2130 MachineFunction *MF = MI.getParent()->getParent();
2131 ApplyRegBankMapping ApplyBank(*this, MRI, DstBank);
2132 GISelObserverWrapper Observer(&ApplyBank);
2133 MachineIRBuilder B(MI);
2134 LegalizerHelper Helper(*MF, Observer, B);
2136 if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized)
2137 llvm_unreachable("widen scalar should have succeeded");
2141 case AMDGPU::G_ICMP:
2142 case AMDGPU::G_UADDO:
2143 case AMDGPU::G_USUBO:
2144 case AMDGPU::G_UADDE:
2145 case AMDGPU::G_SADDE:
2146 case AMDGPU::G_USUBE:
2147 case AMDGPU::G_SSUBE: {
2148 unsigned BoolDstOp = Opc == AMDGPU::G_ICMP ? 0 : 1;
2149 Register DstReg = MI.getOperand(BoolDstOp).getReg();
2151 const RegisterBank *DstBank =
2152 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2153 if (DstBank != &AMDGPU::SGPRRegBank)
2156 const bool HasCarryIn = MI.getNumOperands() == 5;
2158 // If this is a scalar compare, promote the result to s32, as the selection
2159 // will end up using a copy to a 32-bit vreg.
2160 const LLT S32 = LLT::scalar(32);
2161 Register NewDstReg = MRI.createGenericVirtualRegister(S32);
2162 MRI.setRegBank(NewDstReg, AMDGPU::SGPRRegBank);
2163 MI.getOperand(BoolDstOp).setReg(NewDstReg);
2164 MachineIRBuilder B(MI);
2167 Register NewSrcReg = MRI.createGenericVirtualRegister(S32);
2168 MRI.setRegBank(NewSrcReg, AMDGPU::SGPRRegBank);
2169 B.buildZExt(NewSrcReg, MI.getOperand(4).getReg());
2170 MI.getOperand(4).setReg(NewSrcReg);
2173 MachineBasicBlock *MBB = MI.getParent();
2174 B.setInsertPt(*MBB, std::next(MI.getIterator()));
2176 // If we had a constrained VCC result register, a copy was inserted to VCC
2178 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(0));
2179 if (DefRegs.empty())
2180 DefRegs.push_back(DstReg);
2181 B.buildTrunc(DefRegs[0], NewDstReg);
2184 case AMDGPU::G_SELECT: {
2185 Register DstReg = MI.getOperand(0).getReg();
2186 LLT DstTy = MRI.getType(DstReg);
2188 SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(1));
2189 if (CondRegs.empty())
2190 CondRegs.push_back(MI.getOperand(1).getReg());
2192 assert(CondRegs.size() == 1);
2195 const RegisterBank *CondBank = getRegBank(CondRegs[0], MRI, *TRI);
2196 if (CondBank == &AMDGPU::SGPRRegBank) {
2197 MachineIRBuilder B(MI);
2198 const LLT S32 = LLT::scalar(32);
2199 Register NewCondReg = MRI.createGenericVirtualRegister(S32);
2200 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank);
2202 MI.getOperand(1).setReg(NewCondReg);
2203 B.buildZExt(NewCondReg, CondRegs[0]);
2206 if (DstTy.getSizeInBits() != 64)
2209 MachineIRBuilder B(MI);
2210 LLT HalfTy = getHalfSizedType(DstTy);
2212 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2213 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2214 SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(3));
2216 // All inputs are SGPRs, nothing special to do.
2217 if (DefRegs.empty()) {
2218 assert(Src1Regs.empty() && Src2Regs.empty());
2222 if (Src1Regs.empty())
2223 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2225 setRegsToType(MRI, Src1Regs, HalfTy);
2228 if (Src2Regs.empty())
2229 split64BitValueForMapping(B, Src2Regs, HalfTy, MI.getOperand(3).getReg());
2231 setRegsToType(MRI, Src2Regs, HalfTy);
2233 setRegsToType(MRI, DefRegs, HalfTy);
2235 B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0]);
2236 B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1]);
2238 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2239 MI.eraseFromParent();
2242 case AMDGPU::G_BRCOND: {
2243 Register CondReg = MI.getOperand(0).getReg();
2244 // FIXME: Should use legalizer helper, but should change bool ext type.
2245 const RegisterBank *CondBank =
2246 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2248 if (CondBank == &AMDGPU::SGPRRegBank) {
2249 MachineIRBuilder B(MI);
2250 const LLT S32 = LLT::scalar(32);
2251 Register NewCondReg = MRI.createGenericVirtualRegister(S32);
2252 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank);
2254 MI.getOperand(0).setReg(NewCondReg);
2255 B.buildZExt(NewCondReg, CondReg);
2263 case AMDGPU::G_XOR: {
2264 // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
2265 // there is a VGPR input.
2266 Register DstReg = MI.getOperand(0).getReg();
2267 LLT DstTy = MRI.getType(DstReg);
2269 if (DstTy.getSizeInBits() == 1) {
2270 const RegisterBank *DstBank =
2271 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2272 if (DstBank == &AMDGPU::VCCRegBank)
2275 MachineFunction *MF = MI.getParent()->getParent();
2276 ApplyRegBankMapping ApplyBank(*this, MRI, DstBank);
2277 GISelObserverWrapper Observer(&ApplyBank);
2278 MachineIRBuilder B(MI);
2279 LegalizerHelper Helper(*MF, Observer, B);
2281 if (Helper.widenScalar(MI, 0, LLT::scalar(32)) !=
2282 LegalizerHelper::Legalized)
2283 llvm_unreachable("widen scalar should have succeeded");
2287 if (DstTy.getSizeInBits() != 64)
2290 LLT HalfTy = getHalfSizedType(DstTy);
2291 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2292 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(1));
2293 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2295 // All inputs are SGPRs, nothing special to do.
2296 if (DefRegs.empty()) {
2297 assert(Src0Regs.empty() && Src1Regs.empty());
2301 assert(DefRegs.size() == 2);
2302 assert(Src0Regs.size() == Src1Regs.size() &&
2303 (Src0Regs.empty() || Src0Regs.size() == 2));
2305 // Depending on where the source registers came from, the generic code may
2306 // have decided to split the inputs already or not. If not, we still need to
2307 // extract the values.
2308 MachineIRBuilder B(MI);
2310 if (Src0Regs.empty())
2311 split64BitValueForMapping(B, Src0Regs, HalfTy, MI.getOperand(1).getReg());
2313 setRegsToType(MRI, Src0Regs, HalfTy);
2315 if (Src1Regs.empty())
2316 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2318 setRegsToType(MRI, Src1Regs, HalfTy);
2320 setRegsToType(MRI, DefRegs, HalfTy);
2324 .addUse(Src0Regs[0])
2325 .addUse(Src1Regs[0]);
2329 .addUse(Src0Regs[1])
2330 .addUse(Src1Regs[1]);
2332 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2333 MI.eraseFromParent();
2340 case AMDGPU::G_LSHR:
2341 case AMDGPU::G_ASHR: {
2342 Register DstReg = MI.getOperand(0).getReg();
2343 LLT DstTy = MRI.getType(DstReg);
2345 // 16-bit operations are VALU only, but can be promoted to 32-bit SALU.
2346 // Packed 16-bit operations need to be scalarized and promoted.
2347 if (DstTy != LLT::scalar(16) && DstTy != LLT::vector(2, 16))
2350 const RegisterBank *DstBank =
2351 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2352 if (DstBank == &AMDGPU::VGPRRegBank)
2355 const LLT S32 = LLT::scalar(32);
2356 MachineBasicBlock *MBB = MI.getParent();
2357 MachineFunction *MF = MBB->getParent();
2358 MachineIRBuilder B(MI);
2359 ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank);
2360 GISelObserverWrapper Observer(&ApplySALU);
2362 if (DstTy.isVector()) {
2363 B.setChangeObserver(Observer);
2365 Register WideSrc0Lo, WideSrc0Hi;
2366 Register WideSrc1Lo, WideSrc1Hi;
2368 std::tie(WideSrc0Lo, WideSrc0Hi)
2369 = unpackV2S16ToS32(B, MI.getOperand(1).getReg(), AMDGPU::G_ANYEXT);
2370 std::tie(WideSrc1Lo, WideSrc1Hi)
2371 = unpackV2S16ToS32(B, MI.getOperand(2).getReg(), AMDGPU::G_ANYEXT);
2372 auto Lo = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Lo, WideSrc1Lo});
2373 auto Hi = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Hi, WideSrc1Hi});
2374 B.buildBuildVectorTrunc(DstReg, {Lo.getReg(0), Hi.getReg(0)});
2375 MI.eraseFromParent();
2377 LegalizerHelper Helper(*MF, Observer, B);
2379 if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized)
2380 llvm_unreachable("widen scalar should have succeeded");
2382 // FIXME: s16 shift amounts should be legal.
2383 if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR ||
2384 Opc == AMDGPU::G_ASHR) {
2385 B.setInsertPt(*MBB, MI.getIterator());
2386 if (Helper.widenScalar(MI, 1, S32) != LegalizerHelper::Legalized)
2387 llvm_unreachable("widen scalar should have succeeded");
2393 case AMDGPU::G_SMIN:
2394 case AMDGPU::G_SMAX:
2395 case AMDGPU::G_UMIN:
2396 case AMDGPU::G_UMAX: {
2397 Register DstReg = MI.getOperand(0).getReg();
2398 const RegisterBank *DstBank =
2399 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2400 if (DstBank == &AMDGPU::VGPRRegBank)
2403 MachineFunction *MF = MI.getParent()->getParent();
2404 MachineIRBuilder B(MI);
2406 // Turn scalar min/max into a compare and select.
2407 LLT Ty = MRI.getType(DstReg);
2408 const LLT S32 = LLT::scalar(32);
2409 const LLT S16 = LLT::scalar(16);
2410 const LLT V2S16 = LLT::vector(2, 16);
2413 ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank);
2414 GISelObserverWrapper Observer(&ApplySALU);
2415 B.setChangeObserver(Observer);
2417 // Need to widen to s32, and expand as cmp + select, and avoid producing
2418 // illegal vector extends or unmerges that would need further
2421 // TODO: Should we just readfirstlane? That should probably be handled
2422 // with a UniformVGPR register bank that wouldn't need special
2423 // consideration here.
2425 Register Dst = MI.getOperand(0).getReg();
2426 Register Src0 = MI.getOperand(1).getReg();
2427 Register Src1 = MI.getOperand(2).getReg();
2429 Register WideSrc0Lo, WideSrc0Hi;
2430 Register WideSrc1Lo, WideSrc1Hi;
2432 unsigned ExtendOp = minMaxToExtend(MI.getOpcode());
2434 std::tie(WideSrc0Lo, WideSrc0Hi) = unpackV2S16ToS32(B, Src0, ExtendOp);
2435 std::tie(WideSrc1Lo, WideSrc1Hi) = unpackV2S16ToS32(B, Src1, ExtendOp);
2437 Register Lo = MRI.createGenericVirtualRegister(S32);
2438 Register Hi = MRI.createGenericVirtualRegister(S32);
2439 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
2440 buildExpandedScalarMinMax(B, Pred, Lo, WideSrc0Lo, WideSrc1Lo);
2441 buildExpandedScalarMinMax(B, Pred, Hi, WideSrc0Hi, WideSrc1Hi);
2443 B.buildBuildVectorTrunc(Dst, {Lo, Hi});
2444 MI.eraseFromParent();
2445 } else if (Ty == S16) {
2446 ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank);
2447 GISelObserverWrapper Observer(&ApplySALU);
2448 LegalizerHelper Helper(*MF, Observer, B);
2450 // Need to widen to s32, and expand as cmp + select.
2451 if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized)
2452 llvm_unreachable("widenScalar should have succeeded");
2454 // FIXME: This is relying on widenScalar leaving MI in place.
2455 lowerScalarMinMax(B, MI);
2457 lowerScalarMinMax(B, MI);
2461 case AMDGPU::G_SEXT_INREG: {
2462 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(1));
2463 if (SrcRegs.empty())
2464 break; // Nothing to repair
2466 const LLT S32 = LLT::scalar(32);
2467 MachineIRBuilder B(MI);
2468 ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank);
2469 GISelObserverWrapper Observer(&O);
2470 B.setChangeObserver(Observer);
2472 // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs
2473 // we would need to further expand, and doesn't let us directly set the
2474 // result registers.
2475 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
2477 int Amt = MI.getOperand(2).getImm();
2480 // The low bits are unchanged.
2481 B.buildCopy(DstRegs[0], SrcRegs[0]);
2483 // Extend in the low bits and propagate the sign bit to the high half.
2484 B.buildSExtInReg(DstRegs[0], SrcRegs[0], Amt);
2487 B.buildAShr(DstRegs[1], DstRegs[0], B.buildConstant(S32, 31));
2489 // The low bits are unchanged, and extend in the high bits.
2490 B.buildCopy(DstRegs[0], SrcRegs[0]);
2491 B.buildSExtInReg(DstRegs[1], DstRegs[0], Amt - 32);
2494 Register DstReg = MI.getOperand(0).getReg();
2495 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2496 MI.eraseFromParent();
2499 case AMDGPU::G_CTPOP:
2500 case AMDGPU::G_CTLZ_ZERO_UNDEF:
2501 case AMDGPU::G_CTTZ_ZERO_UNDEF: {
2502 MachineIRBuilder B(MI);
2503 MachineFunction &MF = B.getMF();
2505 const RegisterBank *DstBank =
2506 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2507 if (DstBank == &AMDGPU::SGPRRegBank)
2510 Register SrcReg = MI.getOperand(1).getReg();
2511 const LLT S32 = LLT::scalar(32);
2512 LLT Ty = MRI.getType(SrcReg);
2516 ApplyRegBankMapping ApplyVALU(*this, MRI, &AMDGPU::VGPRRegBank);
2517 GISelObserverWrapper Observer(&ApplyVALU);
2518 LegalizerHelper Helper(MF, Observer, B);
2520 if (Helper.narrowScalar(MI, 1, S32) != LegalizerHelper::Legalized)
2521 llvm_unreachable("narrowScalar should have succeeded");
2524 case AMDGPU::G_SEXT:
2525 case AMDGPU::G_ZEXT:
2526 case AMDGPU::G_ANYEXT: {
2527 Register SrcReg = MI.getOperand(1).getReg();
2528 LLT SrcTy = MRI.getType(SrcReg);
2529 const bool Signed = Opc == AMDGPU::G_SEXT;
2531 assert(empty(OpdMapper.getVRegs(1)));
2533 MachineIRBuilder B(MI);
2534 const RegisterBank *SrcBank =
2535 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2537 Register DstReg = MI.getOperand(0).getReg();
2538 LLT DstTy = MRI.getType(DstReg);
2539 if (DstTy.isScalar() &&
2540 SrcBank != &AMDGPU::SGPRRegBank &&
2541 SrcBank != &AMDGPU::VCCRegBank &&
2542 // FIXME: Should handle any type that round to s64 when irregular
2543 // breakdowns supported.
2544 DstTy.getSizeInBits() == 64 &&
2545 SrcTy.getSizeInBits() <= 32) {
2546 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2548 // Extend to 32-bit, and then extend the low half.
2550 // TODO: Should really be buildSExtOrCopy
2551 B.buildSExtOrTrunc(DefRegs[0], SrcReg);
2552 } else if (Opc == AMDGPU::G_ZEXT) {
2553 B.buildZExtOrTrunc(DefRegs[0], SrcReg);
2555 B.buildAnyExtOrTrunc(DefRegs[0], SrcReg);
2558 extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank);
2559 MRI.setRegBank(DstReg, *SrcBank);
2560 MI.eraseFromParent();
2564 if (SrcTy != LLT::scalar(1))
2567 // It is not legal to have a legalization artifact with a VCC source. Rather
2568 // than introducing a copy, insert the select we would have to select the
2570 if (SrcBank == &AMDGPU::VCCRegBank) {
2571 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2573 const RegisterBank *DstBank = &AMDGPU::VGPRRegBank;
2575 unsigned DstSize = DstTy.getSizeInBits();
2576 // 64-bit select is SGPR only
2577 const bool UseSel64 = DstSize > 32 &&
2578 SrcBank->getID() == AMDGPU::SGPRRegBankID;
2580 // TODO: Should s16 select be legal?
2581 LLT SelType = UseSel64 ? LLT::scalar(64) : LLT::scalar(32);
2582 auto True = B.buildConstant(SelType, Signed ? -1 : 1);
2583 auto False = B.buildConstant(SelType, 0);
2585 MRI.setRegBank(True.getReg(0), *DstBank);
2586 MRI.setRegBank(False.getReg(0), *DstBank);
2587 MRI.setRegBank(DstReg, *DstBank);
2590 B.buildSelect(DefRegs[0], SrcReg, True, False);
2591 extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank, true);
2592 } else if (DstSize < 32) {
2593 auto Sel = B.buildSelect(SelType, SrcReg, True, False);
2594 MRI.setRegBank(Sel.getReg(0), *DstBank);
2595 B.buildTrunc(DstReg, Sel);
2597 B.buildSelect(DstReg, SrcReg, True, False);
2600 MI.eraseFromParent();
2606 case AMDGPU::G_BUILD_VECTOR:
2607 case AMDGPU::G_BUILD_VECTOR_TRUNC: {
2608 Register DstReg = MI.getOperand(0).getReg();
2609 LLT DstTy = MRI.getType(DstReg);
2610 if (DstTy != LLT::vector(2, 16))
2613 assert(MI.getNumOperands() == 3 && OpdMapper.getVRegs(0).empty());
2614 substituteSimpleCopyRegs(OpdMapper, 1);
2615 substituteSimpleCopyRegs(OpdMapper, 2);
2617 const RegisterBank *DstBank =
2618 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2619 if (DstBank == &AMDGPU::SGPRRegBank)
2620 break; // Can use S_PACK_* instructions.
2622 MachineIRBuilder B(MI);
2624 Register Lo = MI.getOperand(1).getReg();
2625 Register Hi = MI.getOperand(2).getReg();
2626 const LLT S32 = LLT::scalar(32);
2628 const RegisterBank *BankLo =
2629 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2630 const RegisterBank *BankHi =
2631 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2636 if (Opc == AMDGPU::G_BUILD_VECTOR) {
2637 ZextLo = B.buildZExt(S32, Lo).getReg(0);
2638 MRI.setRegBank(ZextLo, *BankLo);
2640 Register ZextHi = B.buildZExt(S32, Hi).getReg(0);
2641 MRI.setRegBank(ZextHi, *BankHi);
2643 auto ShiftAmt = B.buildConstant(S32, 16);
2644 MRI.setRegBank(ShiftAmt.getReg(0), *BankHi);
2646 ShiftHi = B.buildShl(S32, ZextHi, ShiftAmt).getReg(0);
2647 MRI.setRegBank(ShiftHi, *BankHi);
2649 Register MaskLo = B.buildConstant(S32, 0xffff).getReg(0);
2650 MRI.setRegBank(MaskLo, *BankLo);
2652 auto ShiftAmt = B.buildConstant(S32, 16);
2653 MRI.setRegBank(ShiftAmt.getReg(0), *BankHi);
2655 ShiftHi = B.buildShl(S32, Hi, ShiftAmt).getReg(0);
2656 MRI.setRegBank(ShiftHi, *BankHi);
2658 ZextLo = B.buildAnd(S32, Lo, MaskLo).getReg(0);
2659 MRI.setRegBank(ZextLo, *BankLo);
2662 auto Or = B.buildOr(S32, ZextLo, ShiftHi);
2663 MRI.setRegBank(Or.getReg(0), *DstBank);
2665 B.buildBitcast(DstReg, Or);
2666 MI.eraseFromParent();
2669 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
2670 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
2672 assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty());
2674 Register DstReg = MI.getOperand(0).getReg();
2675 Register SrcReg = MI.getOperand(1).getReg();
2677 const LLT S32 = LLT::scalar(32);
2678 LLT DstTy = MRI.getType(DstReg);
2679 LLT SrcTy = MRI.getType(SrcReg);
2681 if (foldExtractEltToCmpSelect(MI, MRI, OpdMapper))
2684 MachineIRBuilder B(MI);
2686 const ValueMapping &DstMapping
2687 = OpdMapper.getInstrMapping().getOperandMapping(0);
2688 const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank;
2689 const RegisterBank *SrcBank =
2690 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2691 const RegisterBank *IdxBank =
2692 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2694 Register BaseIdxReg;
2695 unsigned ConstOffset;
2696 MachineInstr *OffsetDef;
2697 std::tie(BaseIdxReg, ConstOffset, OffsetDef) =
2698 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(2).getReg());
2700 // See if the index is an add of a constant which will be foldable by moving
2701 // the base register of the index later if this is going to be executed in a
2702 // waterfall loop. This is essentially to reassociate the add of a constant
2703 // with the readfirstlane.
2704 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2706 ConstOffset < SrcTy.getNumElements();
2708 // Move the base register. We'll re-insert the add later.
2709 if (ShouldMoveIndexIntoLoop)
2710 MI.getOperand(2).setReg(BaseIdxReg);
2712 // If this is a VGPR result only because the index was a VGPR result, the
2713 // actual indexing will be done on the SGPR source vector, which will
2714 // produce a scalar result. We need to copy to the VGPR result inside the
2716 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank &&
2717 SrcBank == &AMDGPU::SGPRRegBank;
2718 if (DstRegs.empty()) {
2719 applyDefaultMapping(OpdMapper);
2721 executeInWaterfallLoop(MI, MRI, { 2 });
2723 if (NeedCopyToVGPR) {
2724 // We don't want a phi for this temporary reg.
2725 Register TmpReg = MRI.createGenericVirtualRegister(DstTy);
2726 MRI.setRegBank(TmpReg, AMDGPU::SGPRRegBank);
2727 MI.getOperand(0).setReg(TmpReg);
2728 B.setInsertPt(*MI.getParent(), ++MI.getIterator());
2730 // Use a v_mov_b32 here to make the exec dependency explicit.
2731 buildVCopy(B, DstReg, TmpReg);
2734 // Re-insert the constant offset add inside the waterfall loop.
2735 if (ShouldMoveIndexIntoLoop)
2736 reinsertVectorIndexAdd(B, MI, 2, ConstOffset);
2741 assert(DstTy.getSizeInBits() == 64);
2743 LLT Vec32 = LLT::vector(2 * SrcTy.getNumElements(), 32);
2745 auto CastSrc = B.buildBitcast(Vec32, SrcReg);
2746 auto One = B.buildConstant(S32, 1);
2748 MachineBasicBlock::iterator MII = MI.getIterator();
2750 // Split the vector index into 32-bit pieces. Prepare to move all of the
2751 // new instructions into a waterfall loop if necessary.
2753 // Don't put the bitcast or constant in the loop.
2754 MachineInstrSpan Span(MII, &B.getMBB());
2756 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2757 auto IdxLo = B.buildShl(S32, BaseIdxReg, One);
2758 auto IdxHi = B.buildAdd(S32, IdxLo, One);
2760 auto Extract0 = B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo);
2761 auto Extract1 = B.buildExtractVectorElement(DstRegs[1], CastSrc, IdxHi);
2763 MRI.setRegBank(DstReg, *DstBank);
2764 MRI.setRegBank(CastSrc.getReg(0), *SrcBank);
2765 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank);
2766 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank);
2767 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank);
2769 SmallSet<Register, 4> OpsToWaterfall;
2770 if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 2 })) {
2771 MI.eraseFromParent();
2775 // Remove the original instruction to avoid potentially confusing the
2776 // waterfall loop logic.
2777 B.setInstr(*Span.begin());
2778 MI.eraseFromParent();
2779 executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()),
2780 OpsToWaterfall, MRI);
2782 if (NeedCopyToVGPR) {
2783 MachineBasicBlock *LoopBB = Extract1->getParent();
2784 Register TmpReg0 = MRI.createGenericVirtualRegister(S32);
2785 Register TmpReg1 = MRI.createGenericVirtualRegister(S32);
2786 MRI.setRegBank(TmpReg0, AMDGPU::SGPRRegBank);
2787 MRI.setRegBank(TmpReg1, AMDGPU::SGPRRegBank);
2789 Extract0->getOperand(0).setReg(TmpReg0);
2790 Extract1->getOperand(0).setReg(TmpReg1);
2792 B.setInsertPt(*LoopBB, ++Extract1->getIterator());
2794 buildVCopy(B, DstRegs[0], TmpReg0);
2795 buildVCopy(B, DstRegs[1], TmpReg1);
2798 if (ShouldMoveIndexIntoLoop)
2799 reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset);
2803 case AMDGPU::G_INSERT_VECTOR_ELT: {
2804 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2));
2806 Register DstReg = MI.getOperand(0).getReg();
2807 LLT VecTy = MRI.getType(DstReg);
2809 assert(OpdMapper.getVRegs(0).empty());
2810 assert(OpdMapper.getVRegs(3).empty());
2812 if (substituteSimpleCopyRegs(OpdMapper, 1))
2813 MRI.setType(MI.getOperand(1).getReg(), VecTy);
2815 if (foldInsertEltToCmpSelect(MI, MRI, OpdMapper))
2818 const RegisterBank *IdxBank =
2819 OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank;
2821 Register SrcReg = MI.getOperand(1).getReg();
2822 Register InsReg = MI.getOperand(2).getReg();
2823 LLT InsTy = MRI.getType(InsReg);
2826 Register BaseIdxReg;
2827 unsigned ConstOffset;
2828 MachineInstr *OffsetDef;
2829 std::tie(BaseIdxReg, ConstOffset, OffsetDef) =
2830 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(3).getReg());
2832 // See if the index is an add of a constant which will be foldable by moving
2833 // the base register of the index later if this is going to be executed in a
2834 // waterfall loop. This is essentially to reassociate the add of a constant
2835 // with the readfirstlane.
2836 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2838 ConstOffset < VecTy.getNumElements();
2840 // Move the base register. We'll re-insert the add later.
2841 if (ShouldMoveIndexIntoLoop)
2842 MI.getOperand(3).setReg(BaseIdxReg);
2845 if (InsRegs.empty()) {
2846 executeInWaterfallLoop(MI, MRI, { 3 });
2848 // Re-insert the constant offset add inside the waterfall loop.
2849 if (ShouldMoveIndexIntoLoop) {
2850 MachineIRBuilder B(MI);
2851 reinsertVectorIndexAdd(B, MI, 3, ConstOffset);
2858 assert(InsTy.getSizeInBits() == 64);
2860 const LLT S32 = LLT::scalar(32);
2861 LLT Vec32 = LLT::vector(2 * VecTy.getNumElements(), 32);
2863 MachineIRBuilder B(MI);
2864 auto CastSrc = B.buildBitcast(Vec32, SrcReg);
2865 auto One = B.buildConstant(S32, 1);
2867 // Split the vector index into 32-bit pieces. Prepare to move all of the
2868 // new instructions into a waterfall loop if necessary.
2870 // Don't put the bitcast or constant in the loop.
2871 MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB());
2873 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2874 auto IdxLo = B.buildShl(S32, BaseIdxReg, One);
2875 auto IdxHi = B.buildAdd(S32, IdxLo, One);
2877 auto InsLo = B.buildInsertVectorElement(Vec32, CastSrc, InsRegs[0], IdxLo);
2878 auto InsHi = B.buildInsertVectorElement(Vec32, InsLo, InsRegs[1], IdxHi);
2880 const RegisterBank *DstBank =
2881 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
2882 const RegisterBank *SrcBank =
2883 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank;
2884 const RegisterBank *InsSrcBank =
2885 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2887 MRI.setRegBank(InsReg, *InsSrcBank);
2888 MRI.setRegBank(CastSrc.getReg(0), *SrcBank);
2889 MRI.setRegBank(InsLo.getReg(0), *DstBank);
2890 MRI.setRegBank(InsHi.getReg(0), *DstBank);
2891 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank);
2892 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank);
2893 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank);
2896 SmallSet<Register, 4> OpsToWaterfall;
2897 if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 3 })) {
2898 B.setInsertPt(B.getMBB(), MI);
2899 B.buildBitcast(DstReg, InsHi);
2900 MI.eraseFromParent();
2904 B.setInstr(*Span.begin());
2905 MI.eraseFromParent();
2907 // Figure out the point after the waterfall loop before mangling the control
2909 executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()),
2910 OpsToWaterfall, MRI);
2912 // The insertion point is now right after the original instruction.
2914 // Keep the bitcast to the original vector type out of the loop. Doing this
2915 // saved an extra phi we don't need inside the loop.
2916 B.buildBitcast(DstReg, InsHi);
2918 // Re-insert the constant offset add inside the waterfall loop.
2919 if (ShouldMoveIndexIntoLoop)
2920 reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset);
2924 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
2925 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
2926 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
2927 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
2928 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
2929 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
2930 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
2931 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
2932 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
2933 case AMDGPU::G_AMDGPU_BUFFER_STORE:
2934 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
2935 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
2936 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
2937 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16:
2938 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
2939 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: {
2940 applyDefaultMapping(OpdMapper);
2941 executeInWaterfallLoop(MI, MRI, {1, 4});
2944 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
2945 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
2946 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
2947 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
2948 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
2949 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
2950 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
2951 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
2952 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
2953 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
2954 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
2955 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: {
2956 applyDefaultMapping(OpdMapper);
2957 executeInWaterfallLoop(MI, MRI, {2, 5});
2960 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
2961 applyDefaultMapping(OpdMapper);
2962 executeInWaterfallLoop(MI, MRI, {3, 6});
2965 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: {
2966 applyMappingSBufferLoad(OpdMapper);
2969 case AMDGPU::G_INTRINSIC: {
2970 switch (MI.getIntrinsicID()) {
2971 case Intrinsic::amdgcn_readlane: {
2972 substituteSimpleCopyRegs(OpdMapper, 2);
2974 assert(OpdMapper.getVRegs(0).empty());
2975 assert(OpdMapper.getVRegs(3).empty());
2977 // Make sure the index is an SGPR. It doesn't make sense to run this in a
2978 // waterfall loop, so assume it's a uniform value.
2979 constrainOpWithReadfirstlane(MI, MRI, 3); // Index
2982 case Intrinsic::amdgcn_writelane: {
2983 assert(OpdMapper.getVRegs(0).empty());
2984 assert(OpdMapper.getVRegs(2).empty());
2985 assert(OpdMapper.getVRegs(3).empty());
2987 substituteSimpleCopyRegs(OpdMapper, 4); // VGPR input val
2988 constrainOpWithReadfirstlane(MI, MRI, 2); // Source value
2989 constrainOpWithReadfirstlane(MI, MRI, 3); // Index
2992 case Intrinsic::amdgcn_ballot:
2993 case Intrinsic::amdgcn_interp_p1:
2994 case Intrinsic::amdgcn_interp_p2:
2995 case Intrinsic::amdgcn_interp_mov:
2996 case Intrinsic::amdgcn_interp_p1_f16:
2997 case Intrinsic::amdgcn_interp_p2_f16: {
2998 applyDefaultMapping(OpdMapper);
3000 // Readlane for m0 value, which is always the last operand.
3001 // FIXME: Should this be a waterfall loop instead?
3002 constrainOpWithReadfirstlane(MI, MRI, MI.getNumOperands() - 1); // Index
3005 case Intrinsic::amdgcn_permlane16:
3006 case Intrinsic::amdgcn_permlanex16: {
3007 // Doing a waterfall loop over these wouldn't make any sense.
3008 substituteSimpleCopyRegs(OpdMapper, 2);
3009 substituteSimpleCopyRegs(OpdMapper, 3);
3010 constrainOpWithReadfirstlane(MI, MRI, 4);
3011 constrainOpWithReadfirstlane(MI, MRI, 5);
3014 case Intrinsic::amdgcn_sbfe:
3015 applyMappingBFEIntrinsic(OpdMapper, true);
3017 case Intrinsic::amdgcn_ubfe:
3018 applyMappingBFEIntrinsic(OpdMapper, false);
3023 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3024 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: {
3025 const AMDGPU::RsrcIntrinsic *RSrcIntrin
3026 = AMDGPU::lookupRsrcIntrinsic(MI.getIntrinsicID());
3027 assert(RSrcIntrin && RSrcIntrin->IsImage);
3028 // Non-images can have complications from operands that allow both SGPR
3029 // and VGPR. For now it's too complicated to figure out the final opcode
3030 // to derive the register bank from the MCInstrDesc.
3031 applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg);
3034 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: {
3035 auto IntrID = MI.getIntrinsicID();
3037 case Intrinsic::amdgcn_ds_ordered_add:
3038 case Intrinsic::amdgcn_ds_ordered_swap: {
3039 // This is only allowed to execute with 1 lane, so readfirstlane is safe.
3040 assert(OpdMapper.getVRegs(0).empty());
3041 substituteSimpleCopyRegs(OpdMapper, 3);
3042 constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3045 case Intrinsic::amdgcn_ds_gws_init:
3046 case Intrinsic::amdgcn_ds_gws_barrier:
3047 case Intrinsic::amdgcn_ds_gws_sema_br: {
3048 // Only the first lane is executes, so readfirstlane is safe.
3049 substituteSimpleCopyRegs(OpdMapper, 1);
3050 constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3053 case Intrinsic::amdgcn_ds_gws_sema_v:
3054 case Intrinsic::amdgcn_ds_gws_sema_p:
3055 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
3056 // Only the first lane is executes, so readfirstlane is safe.
3057 constrainOpWithReadfirstlane(MI, MRI, 1); // M0
3060 case Intrinsic::amdgcn_ds_append:
3061 case Intrinsic::amdgcn_ds_consume: {
3062 constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3065 case Intrinsic::amdgcn_s_sendmsg:
3066 case Intrinsic::amdgcn_s_sendmsghalt: {
3067 // FIXME: Should this use a waterfall loop?
3068 constrainOpWithReadfirstlane(MI, MRI, 2); // M0
3071 case Intrinsic::amdgcn_s_setreg: {
3072 constrainOpWithReadfirstlane(MI, MRI, 2);
3076 if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3077 AMDGPU::lookupRsrcIntrinsic(IntrID)) {
3078 // Non-images can have complications from operands that allow both SGPR
3079 // and VGPR. For now it's too complicated to figure out the final opcode
3080 // to derive the register bank from the MCInstrDesc.
3081 if (RSrcIntrin->IsImage) {
3082 applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg);
3092 case AMDGPU::G_LOAD:
3093 case AMDGPU::G_ZEXTLOAD:
3094 case AMDGPU::G_SEXTLOAD: {
3095 if (applyMappingLoad(MI, OpdMapper, MRI))
3099 case AMDGPU::G_DYN_STACKALLOC:
3100 applyMappingDynStackAlloc(MI, OpdMapper, MRI);
3106 return applyDefaultMapping(OpdMapper);
3109 bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const {
3110 const MachineFunction &MF = *MI.getParent()->getParent();
3111 const MachineRegisterInfo &MRI = MF.getRegInfo();
3112 for (unsigned i = 0, e = MI.getNumOperands();i != e; ++i) {
3113 if (!MI.getOperand(i).isReg())
3115 Register Reg = MI.getOperand(i).getReg();
3116 if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
3117 if (Bank->getID() != AMDGPU::SGPRRegBankID)
3124 const RegisterBankInfo::InstructionMapping &
3125 AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const {
3126 const MachineFunction &MF = *MI.getParent()->getParent();
3127 const MachineRegisterInfo &MRI = MF.getRegInfo();
3128 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3130 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3131 const MachineOperand &SrcOp = MI.getOperand(i);
3135 unsigned Size = getSizeInBits(SrcOp.getReg(), MRI, *TRI);
3136 OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3138 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
3139 MI.getNumOperands());
3142 const RegisterBankInfo::InstructionMapping &
3143 AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
3144 const MachineFunction &MF = *MI.getParent()->getParent();
3145 const MachineRegisterInfo &MRI = MF.getRegInfo();
3146 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3148 // Even though we technically could use SGPRs, this would require knowledge of
3149 // the constant bus restriction. Force all sources to VGPR (except for VCC).
3151 // TODO: Unary ops are trivially OK, so accept SGPRs?
3152 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3153 const MachineOperand &Src = MI.getOperand(i);
3157 unsigned Size = getSizeInBits(Src.getReg(), MRI, *TRI);
3158 unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
3159 OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
3162 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
3163 MI.getNumOperands());
3166 const RegisterBankInfo::InstructionMapping &
3167 AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
3168 const MachineFunction &MF = *MI.getParent()->getParent();
3169 const MachineRegisterInfo &MRI = MF.getRegInfo();
3170 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3172 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
3173 const MachineOperand &Op = MI.getOperand(I);
3177 unsigned Size = getSizeInBits(Op.getReg(), MRI, *TRI);
3178 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3181 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
3182 MI.getNumOperands());
3185 const RegisterBankInfo::InstructionMapping &
3186 AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
3187 const MachineInstr &MI,
3188 int RsrcIdx) const {
3189 // The reported argument index is relative to the IR intrinsic call arguments,
3190 // so we need to shift by the number of defs and the intrinsic ID.
3191 RsrcIdx += MI.getNumExplicitDefs() + 1;
3193 const int NumOps = MI.getNumOperands();
3194 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps);
3196 // TODO: Should packed/unpacked D16 difference be reported here as part of
3197 // the value mapping?
3198 for (int I = 0; I != NumOps; ++I) {
3199 if (!MI.getOperand(I).isReg())
3202 Register OpReg = MI.getOperand(I).getReg();
3203 // We replace some dead address operands with $noreg
3207 unsigned Size = getSizeInBits(OpReg, MRI, *TRI);
3209 // FIXME: Probably need a new intrinsic register bank searchable table to
3210 // handle arbitrary intrinsics easily.
3212 // If this has a sampler, it immediately follows rsrc.
3213 const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1;
3216 // If this must be an SGPR, so we must report whatever it is as legal.
3217 unsigned NewBank = getRegBankID(OpReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
3218 OpdsMapping[I] = AMDGPU::getValueMapping(NewBank, Size);
3220 // Some operands must be VGPR, and these are easy to copy to.
3221 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3225 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), NumOps);
3228 /// Return the mapping for a pointer arugment.
3229 const RegisterBankInfo::ValueMapping *
3230 AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI,
3231 Register PtrReg) const {
3232 LLT PtrTy = MRI.getType(PtrReg);
3233 unsigned Size = PtrTy.getSizeInBits();
3234 if (Subtarget.useFlatForGlobal() ||
3235 !SITargetLowering::isFlatGlobalAddrSpace(PtrTy.getAddressSpace()))
3236 return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3238 // If we're using MUBUF instructions for global memory, an SGPR base register
3239 // is possible. Otherwise this needs to be a VGPR.
3240 const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
3241 return AMDGPU::getValueMapping(PtrBank->getID(), Size);
3244 const RegisterBankInfo::InstructionMapping &
3245 AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
3247 const MachineFunction &MF = *MI.getParent()->getParent();
3248 const MachineRegisterInfo &MRI = MF.getRegInfo();
3249 SmallVector<const ValueMapping*, 2> OpdsMapping(2);
3250 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3251 Register PtrReg = MI.getOperand(1).getReg();
3252 LLT PtrTy = MRI.getType(PtrReg);
3253 unsigned AS = PtrTy.getAddressSpace();
3254 unsigned PtrSize = PtrTy.getSizeInBits();
3256 const ValueMapping *ValMapping;
3257 const ValueMapping *PtrMapping;
3259 const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
3261 if (PtrBank == &AMDGPU::SGPRRegBank &&
3262 SITargetLowering::isFlatGlobalAddrSpace(AS)) {
3263 if (isScalarLoadLegal(MI)) {
3264 // We have a uniform instruction so we want to use an SMRD load
3265 ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3266 PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize);
3268 ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3270 // If we're using MUBUF instructions for global memory, an SGPR base
3271 // register is possible. Otherwise this needs to be a VGPR.
3272 unsigned PtrBankID = Subtarget.useFlatForGlobal() ?
3273 AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID;
3275 PtrMapping = AMDGPU::getValueMapping(PtrBankID, PtrSize);
3278 ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3279 PtrMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize);
3282 OpdsMapping[0] = ValMapping;
3283 OpdsMapping[1] = PtrMapping;
3284 const RegisterBankInfo::InstructionMapping &Mapping = getInstructionMapping(
3285 1, 1, getOperandsMapping(OpdsMapping), MI.getNumOperands());
3288 // FIXME: Do we want to add a mapping for FLAT load, or should we just
3289 // handle that during instruction selection?
3293 AMDGPURegisterBankInfo::getRegBankID(Register Reg,
3294 const MachineRegisterInfo &MRI,
3295 const TargetRegisterInfo &TRI,
3296 unsigned Default) const {
3297 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI);
3298 return Bank ? Bank->getID() : Default;
3302 static unsigned regBankUnion(unsigned RB0, unsigned RB1) {
3303 return (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID) ?
3304 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
3307 static int regBankBoolUnion(int RB0, int RB1) {
3316 if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID)
3317 return AMDGPU::VCCRegBankID;
3319 // vcc, vgpr -> vgpr
3320 return regBankUnion(RB0, RB1);
3323 const RegisterBankInfo::ValueMapping *
3324 AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
3325 const MachineRegisterInfo &MRI,
3326 const TargetRegisterInfo &TRI) const {
3327 // Lie and claim anything is legal, even though this needs to be an SGPR
3328 // applyMapping will have to deal with it as a waterfall loop.
3329 unsigned Bank = getRegBankID(Reg, MRI, TRI, AMDGPU::SGPRRegBankID);
3330 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3331 return AMDGPU::getValueMapping(Bank, Size);
3334 const RegisterBankInfo::ValueMapping *
3335 AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
3336 const MachineRegisterInfo &MRI,
3337 const TargetRegisterInfo &TRI) const {
3338 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3339 return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3342 const RegisterBankInfo::ValueMapping *
3343 AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg,
3344 const MachineRegisterInfo &MRI,
3345 const TargetRegisterInfo &TRI) const {
3346 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3347 return AMDGPU::getValueMapping(AMDGPU::AGPRRegBankID, Size);
3351 /// This function must return a legal mapping, because
3352 /// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called
3353 /// in RegBankSelect::Mode::Fast. Any mapping that would cause a
3354 /// VGPR to SGPR generated is illegal.
3356 // Operands that must be SGPRs must accept potentially divergent VGPRs as
3357 // legal. These will be dealt with in applyMappingImpl.
3359 const RegisterBankInfo::InstructionMapping &
3360 AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
3361 const MachineFunction &MF = *MI.getParent()->getParent();
3362 const MachineRegisterInfo &MRI = MF.getRegInfo();
3365 // The default logic bothers to analyze impossible alternative mappings. We
3366 // want the most straightforward mapping, so just directly handle this.
3367 const RegisterBank *DstBank = getRegBank(MI.getOperand(0).getReg(), MRI,
3369 const RegisterBank *SrcBank = getRegBank(MI.getOperand(1).getReg(), MRI,
3371 assert(SrcBank && "src bank should have been assigned already");
3375 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3376 if (cannotCopy(*DstBank, *SrcBank, Size))
3377 return getInvalidInstructionMapping();
3379 const ValueMapping &ValMap = getValueMapping(0, Size, *DstBank);
3380 return getInstructionMapping(
3382 /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1);
3385 if (MI.isRegSequence()) {
3386 // If any input is a VGPR, the result must be a VGPR. The default handling
3387 // assumes any copy between banks is legal.
3388 unsigned BankID = AMDGPU::SGPRRegBankID;
3390 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3391 auto OpBank = getRegBankID(MI.getOperand(I).getReg(), MRI, *TRI);
3392 // It doesn't make sense to use vcc or scc banks here, so just ignore
3394 if (OpBank != AMDGPU::SGPRRegBankID) {
3395 BankID = AMDGPU::VGPRRegBankID;
3399 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3401 const ValueMapping &ValMap = getValueMapping(0, Size, getRegBank(BankID));
3402 return getInstructionMapping(
3404 /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1);
3407 // The default handling is broken and doesn't handle illegal SGPR->VGPR copies
3410 // TODO: There are additional exec masking dependencies to analyze.
3411 if (MI.getOpcode() == TargetOpcode::G_PHI) {
3412 // TODO: Generate proper invalid bank enum.
3413 int ResultBank = -1;
3414 Register DstReg = MI.getOperand(0).getReg();
3416 // Sometimes the result may have already been assigned a bank.
3417 if (const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI))
3418 ResultBank = DstBank->getID();
3420 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3421 Register Reg = MI.getOperand(I).getReg();
3422 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
3424 // FIXME: Assuming VGPR for any undetermined inputs.
3425 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) {
3426 ResultBank = AMDGPU::VGPRRegBankID;
3430 // FIXME: Need to promote SGPR case to s32
3431 unsigned OpBank = Bank->getID();
3432 ResultBank = regBankBoolUnion(ResultBank, OpBank);
3435 assert(ResultBank != -1);
3437 unsigned Size = MRI.getType(DstReg).getSizeInBits();
3439 const ValueMapping &ValMap =
3440 getValueMapping(0, Size, getRegBank(ResultBank));
3441 return getInstructionMapping(
3443 /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1);
3446 const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
3447 if (Mapping.isValid())
3450 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3452 switch (MI.getOpcode()) {
3454 return getInvalidInstructionMapping();
3458 case AMDGPU::G_XOR: {
3459 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3461 const RegisterBank *DstBank
3462 = getRegBank(MI.getOperand(0).getReg(), MRI, *TRI);
3464 unsigned TargetBankID = -1;
3465 unsigned BankLHS = -1;
3466 unsigned BankRHS = -1;
3468 TargetBankID = DstBank->getID();
3469 if (DstBank == &AMDGPU::VCCRegBank) {
3470 TargetBankID = AMDGPU::VCCRegBankID;
3471 BankLHS = AMDGPU::VCCRegBankID;
3472 BankRHS = AMDGPU::VCCRegBankID;
3474 BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
3475 AMDGPU::SGPRRegBankID);
3476 BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
3477 AMDGPU::SGPRRegBankID);
3480 BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
3481 AMDGPU::VCCRegBankID);
3482 BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
3483 AMDGPU::VCCRegBankID);
3485 // Both inputs should be true booleans to produce a boolean result.
3486 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) {
3487 TargetBankID = AMDGPU::VGPRRegBankID;
3488 } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) {
3489 TargetBankID = AMDGPU::VCCRegBankID;
3490 BankLHS = AMDGPU::VCCRegBankID;
3491 BankRHS = AMDGPU::VCCRegBankID;
3492 } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) {
3493 TargetBankID = AMDGPU::SGPRRegBankID;
3497 OpdsMapping[0] = AMDGPU::getValueMapping(TargetBankID, Size);
3498 OpdsMapping[1] = AMDGPU::getValueMapping(BankLHS, Size);
3499 OpdsMapping[2] = AMDGPU::getValueMapping(BankRHS, Size);
3505 if (isSALUMapping(MI)) {
3506 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size);
3507 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
3509 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size);
3510 unsigned Bank1 = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI/*, DefaultBankID*/);
3511 OpdsMapping[1] = AMDGPU::getValueMapping(Bank1, Size);
3513 unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI/*, DefaultBankID*/);
3514 OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size);
3522 case AMDGPU::G_PTR_ADD:
3523 case AMDGPU::G_PTRMASK:
3528 case AMDGPU::G_LSHR:
3529 case AMDGPU::G_ASHR:
3530 case AMDGPU::G_UADDO:
3531 case AMDGPU::G_USUBO:
3532 case AMDGPU::G_UADDE:
3533 case AMDGPU::G_SADDE:
3534 case AMDGPU::G_USUBE:
3535 case AMDGPU::G_SSUBE:
3536 case AMDGPU::G_SMIN:
3537 case AMDGPU::G_SMAX:
3538 case AMDGPU::G_UMIN:
3539 case AMDGPU::G_UMAX:
3540 case AMDGPU::G_SHUFFLE_VECTOR:
3541 if (isSALUMapping(MI))
3542 return getDefaultMappingSOP(MI);
3545 case AMDGPU::G_FADD:
3546 case AMDGPU::G_FSUB:
3547 case AMDGPU::G_FPTOSI:
3548 case AMDGPU::G_FPTOUI:
3549 case AMDGPU::G_FMUL:
3551 case AMDGPU::G_FMAD:
3552 case AMDGPU::G_FSQRT:
3553 case AMDGPU::G_FFLOOR:
3554 case AMDGPU::G_FCEIL:
3555 case AMDGPU::G_FRINT:
3556 case AMDGPU::G_SITOFP:
3557 case AMDGPU::G_UITOFP:
3558 case AMDGPU::G_FPTRUNC:
3559 case AMDGPU::G_FPEXT:
3560 case AMDGPU::G_FEXP2:
3561 case AMDGPU::G_FLOG2:
3562 case AMDGPU::G_FMINNUM:
3563 case AMDGPU::G_FMAXNUM:
3564 case AMDGPU::G_FMINNUM_IEEE:
3565 case AMDGPU::G_FMAXNUM_IEEE:
3566 case AMDGPU::G_FCANONICALIZE:
3567 case AMDGPU::G_INTRINSIC_TRUNC:
3568 case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar?
3569 case AMDGPU::G_FSHR: // TODO: Expand for scalar
3570 case AMDGPU::G_AMDGPU_FFBH_U32:
3571 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
3572 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
3573 case AMDGPU::G_AMDGPU_RCP_IFLAG:
3574 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
3575 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
3576 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
3577 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
3578 return getDefaultMappingVOP(MI);
3579 case AMDGPU::G_UMULH:
3580 case AMDGPU::G_SMULH: {
3581 if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI))
3582 return getDefaultMappingSOP(MI);
3583 return getDefaultMappingVOP(MI);
3585 case AMDGPU::G_IMPLICIT_DEF: {
3586 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3587 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3590 case AMDGPU::G_FCONSTANT:
3591 case AMDGPU::G_CONSTANT:
3592 case AMDGPU::G_GLOBAL_VALUE:
3593 case AMDGPU::G_BLOCK_ADDR:
3594 case AMDGPU::G_READCYCLECOUNTER: {
3595 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3596 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
3599 case AMDGPU::G_FRAME_INDEX: {
3600 // TODO: This should be the same as other constants, but eliminateFrameIndex
3601 // currently assumes VALU uses.
3602 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3603 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3606 case AMDGPU::G_DYN_STACKALLOC: {
3607 // Result is always uniform, and a wave reduction is needed for the source.
3608 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
3609 unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
3610 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32);
3613 case AMDGPU::G_INSERT: {
3614 unsigned BankID = isSALUMapping(MI) ? AMDGPU::SGPRRegBankID :
3615 AMDGPU::VGPRRegBankID;
3616 unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3617 unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
3618 unsigned EltSize = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI);
3619 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
3620 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize);
3621 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, EltSize);
3622 OpdsMapping[3] = nullptr;
3625 case AMDGPU::G_EXTRACT: {
3626 unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
3627 unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3628 unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
3629 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
3630 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize);
3631 OpdsMapping[2] = nullptr;
3634 case AMDGPU::G_BUILD_VECTOR:
3635 case AMDGPU::G_BUILD_VECTOR_TRUNC: {
3636 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3637 if (DstTy == LLT::vector(2, 16)) {
3638 unsigned DstSize = DstTy.getSizeInBits();
3639 unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3640 unsigned Src0BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
3641 unsigned Src1BankID = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
3642 unsigned DstBankID = regBankUnion(Src0BankID, Src1BankID);
3644 OpdsMapping[0] = AMDGPU::getValueMapping(DstBankID, DstSize);
3645 OpdsMapping[1] = AMDGPU::getValueMapping(Src0BankID, SrcSize);
3646 OpdsMapping[2] = AMDGPU::getValueMapping(Src1BankID, SrcSize);
3652 case AMDGPU::G_MERGE_VALUES:
3653 case AMDGPU::G_CONCAT_VECTORS: {
3654 unsigned Bank = isSALUMapping(MI) ?
3655 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
3656 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3657 unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3659 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
3660 // Op1 and Dst should use the same register bank.
3661 for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i)
3662 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, SrcSize);
3665 case AMDGPU::G_BITCAST:
3666 case AMDGPU::G_INTTOPTR:
3667 case AMDGPU::G_PTRTOINT:
3668 case AMDGPU::G_BITREVERSE:
3669 case AMDGPU::G_FABS:
3670 case AMDGPU::G_FNEG: {
3671 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3672 unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
3673 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
3676 case AMDGPU::G_CTLZ_ZERO_UNDEF:
3677 case AMDGPU::G_CTTZ_ZERO_UNDEF:
3678 case AMDGPU::G_CTPOP: {
3679 unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3680 unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
3681 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32);
3683 // This should really be getValueMappingSGPR64Only, but allowing the generic
3684 // code to handle the register split just makes using LegalizerHelper more
3686 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
3689 case AMDGPU::G_TRUNC: {
3690 Register Dst = MI.getOperand(0).getReg();
3691 Register Src = MI.getOperand(1).getReg();
3692 unsigned Bank = getRegBankID(Src, MRI, *TRI);
3693 unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
3694 unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
3695 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
3696 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize);
3699 case AMDGPU::G_ZEXT:
3700 case AMDGPU::G_SEXT:
3701 case AMDGPU::G_ANYEXT:
3702 case AMDGPU::G_SEXT_INREG: {
3703 Register Dst = MI.getOperand(0).getReg();
3704 Register Src = MI.getOperand(1).getReg();
3705 unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
3706 unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
3709 const RegisterBank *SrcBank = getRegBank(Src, MRI, *TRI);
3711 switch (SrcBank->getID()) {
3712 case AMDGPU::SGPRRegBankID:
3713 DstBank = AMDGPU::SGPRRegBankID;
3716 DstBank = AMDGPU::VGPRRegBankID;
3720 // Scalar extend can use 64-bit BFE, but VGPRs require extending to
3721 // 32-bits, and then to 64.
3722 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize);
3723 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(),
3727 case AMDGPU::G_FCMP: {
3728 unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3729 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
3730 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
3731 OpdsMapping[1] = nullptr; // Predicate Operand.
3732 OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size);
3733 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3736 case AMDGPU::G_STORE: {
3737 assert(MI.getOperand(0).isReg());
3738 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3740 // FIXME: We need to specify a different reg bank once scalar stores are
3742 const ValueMapping *ValMapping =
3743 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
3744 OpdsMapping[0] = ValMapping;
3745 OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg());
3748 case AMDGPU::G_ICMP: {
3749 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
3750 unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3752 // See if the result register has already been constrained to vcc, which may
3753 // happen due to control flow intrinsic lowering.
3754 unsigned DstBank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI,
3755 AMDGPU::SGPRRegBankID);
3756 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
3757 unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI);
3759 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
3760 Op2Bank == AMDGPU::SGPRRegBankID &&
3761 Op3Bank == AMDGPU::SGPRRegBankID &&
3762 (Size == 32 || (Size == 64 &&
3763 (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
3764 Subtarget.hasScalarCompareEq64()));
3766 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
3767 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
3769 // TODO: Use 32-bit for scalar output size.
3770 // SCC results will need to be copied to a 32-bit SGPR virtual register.
3771 const unsigned ResultSize = 1;
3773 OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, ResultSize);
3774 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, Size);
3775 OpdsMapping[3] = AMDGPU::getValueMapping(SrcBank, Size);
3778 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
3779 // VGPR index can be used for waterfall when indexing a SGPR vector.
3780 unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
3781 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3782 unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3783 unsigned IdxSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3784 unsigned IdxBank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
3785 unsigned OutputBankID = regBankUnion(SrcBankID, IdxBank);
3787 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(OutputBankID, DstSize);
3788 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, SrcSize);
3790 // The index can be either if the source vector is VGPR.
3791 OpdsMapping[2] = AMDGPU::getValueMapping(IdxBank, IdxSize);
3794 case AMDGPU::G_INSERT_VECTOR_ELT: {
3795 unsigned OutputBankID = isSALUMapping(MI) ?
3796 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
3798 unsigned VecSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3799 unsigned InsertSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
3800 unsigned IdxSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits();
3801 unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(),
3803 unsigned IdxBankID = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI);
3805 OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize);
3806 OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize);
3808 // This is a weird case, because we need to break down the mapping based on
3809 // the register bank of a different operand.
3810 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) {
3811 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID,
3814 assert(InsertSize == 32 || InsertSize == 64);
3815 OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize);
3818 // The index can be either if the source vector is VGPR.
3819 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBankID, IdxSize);
3822 case AMDGPU::G_UNMERGE_VALUES: {
3823 unsigned Bank = isSALUMapping(MI) ? AMDGPU::SGPRRegBankID :
3824 AMDGPU::VGPRRegBankID;
3826 // Op1 and Dst should use the same register bank.
3827 // FIXME: Shouldn't this be the default? Why do we need to handle this?
3828 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3829 unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI);
3830 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, Size);
3834 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
3835 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
3836 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
3837 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
3838 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
3839 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
3840 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
3841 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
3842 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
3843 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
3844 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
3845 case AMDGPU::G_AMDGPU_BUFFER_STORE:
3846 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
3847 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
3848 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
3849 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: {
3850 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3853 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3856 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3859 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3862 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3864 // Any remaining operands are immediates and were correctly null
3868 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
3869 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
3870 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
3871 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
3872 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
3873 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
3874 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
3875 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
3876 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
3877 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
3878 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
3879 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: {
3881 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3884 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3887 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3890 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3893 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3896 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
3898 // Any remaining operands are immediates and were correctly null
3902 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
3904 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
3907 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3910 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3913 OpdsMapping[3] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
3916 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
3919 OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
3922 OpdsMapping[6] = getSGPROpMapping(MI.getOperand(6).getReg(), MRI, *TRI);
3924 // Any remaining operands are immediates and were correctly null
3928 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: {
3929 // Lie and claim everything is legal, even though some need to be
3930 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
3931 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
3932 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
3934 // We need to convert this to a MUBUF if either the resource of offset is
3936 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
3937 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
3938 unsigned ResultBank = regBankUnion(RSrcBank, OffsetBank);
3940 unsigned Size0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3941 OpdsMapping[0] = AMDGPU::getValueMapping(ResultBank, Size0);
3944 case AMDGPU::G_INTRINSIC: {
3945 switch (MI.getIntrinsicID()) {
3947 return getInvalidInstructionMapping();
3948 case Intrinsic::amdgcn_div_fmas:
3949 case Intrinsic::amdgcn_div_fixup:
3950 case Intrinsic::amdgcn_trig_preop:
3951 case Intrinsic::amdgcn_sin:
3952 case Intrinsic::amdgcn_cos:
3953 case Intrinsic::amdgcn_log_clamp:
3954 case Intrinsic::amdgcn_rcp:
3955 case Intrinsic::amdgcn_rcp_legacy:
3956 case Intrinsic::amdgcn_sqrt:
3957 case Intrinsic::amdgcn_rsq:
3958 case Intrinsic::amdgcn_rsq_legacy:
3959 case Intrinsic::amdgcn_rsq_clamp:
3960 case Intrinsic::amdgcn_fmul_legacy:
3961 case Intrinsic::amdgcn_ldexp:
3962 case Intrinsic::amdgcn_frexp_mant:
3963 case Intrinsic::amdgcn_frexp_exp:
3964 case Intrinsic::amdgcn_fract:
3965 case Intrinsic::amdgcn_cvt_pkrtz:
3966 case Intrinsic::amdgcn_cvt_pknorm_i16:
3967 case Intrinsic::amdgcn_cvt_pknorm_u16:
3968 case Intrinsic::amdgcn_cvt_pk_i16:
3969 case Intrinsic::amdgcn_cvt_pk_u16:
3970 case Intrinsic::amdgcn_fmed3:
3971 case Intrinsic::amdgcn_cubeid:
3972 case Intrinsic::amdgcn_cubema:
3973 case Intrinsic::amdgcn_cubesc:
3974 case Intrinsic::amdgcn_cubetc:
3975 case Intrinsic::amdgcn_sffbh:
3976 case Intrinsic::amdgcn_fmad_ftz:
3977 case Intrinsic::amdgcn_mbcnt_lo:
3978 case Intrinsic::amdgcn_mbcnt_hi:
3979 case Intrinsic::amdgcn_mul_u24:
3980 case Intrinsic::amdgcn_mul_i24:
3981 case Intrinsic::amdgcn_lerp:
3982 case Intrinsic::amdgcn_sad_u8:
3983 case Intrinsic::amdgcn_msad_u8:
3984 case Intrinsic::amdgcn_sad_hi_u8:
3985 case Intrinsic::amdgcn_sad_u16:
3986 case Intrinsic::amdgcn_qsad_pk_u16_u8:
3987 case Intrinsic::amdgcn_mqsad_pk_u16_u8:
3988 case Intrinsic::amdgcn_mqsad_u32_u8:
3989 case Intrinsic::amdgcn_cvt_pk_u8_f32:
3990 case Intrinsic::amdgcn_alignbit:
3991 case Intrinsic::amdgcn_alignbyte:
3992 case Intrinsic::amdgcn_fdot2:
3993 case Intrinsic::amdgcn_sdot2:
3994 case Intrinsic::amdgcn_udot2:
3995 case Intrinsic::amdgcn_sdot4:
3996 case Intrinsic::amdgcn_udot4:
3997 case Intrinsic::amdgcn_sdot8:
3998 case Intrinsic::amdgcn_udot8:
3999 return getDefaultMappingVOP(MI);
4000 case Intrinsic::amdgcn_sbfe:
4001 case Intrinsic::amdgcn_ubfe:
4002 if (isSALUMapping(MI))
4003 return getDefaultMappingSOP(MI);
4004 return getDefaultMappingVOP(MI);
4005 case Intrinsic::amdgcn_ds_swizzle:
4006 case Intrinsic::amdgcn_ds_permute:
4007 case Intrinsic::amdgcn_ds_bpermute:
4008 case Intrinsic::amdgcn_update_dpp:
4009 case Intrinsic::amdgcn_mov_dpp8:
4010 case Intrinsic::amdgcn_mov_dpp:
4011 case Intrinsic::amdgcn_wwm:
4012 case Intrinsic::amdgcn_wqm:
4013 case Intrinsic::amdgcn_softwqm:
4014 return getDefaultMappingAllVGPR(MI);
4015 case Intrinsic::amdgcn_kernarg_segment_ptr:
4016 case Intrinsic::amdgcn_s_getpc:
4017 case Intrinsic::amdgcn_groupstaticsize: {
4018 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4019 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4022 case Intrinsic::amdgcn_wqm_vote: {
4023 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4024 OpdsMapping[0] = OpdsMapping[2]
4025 = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size);
4028 case Intrinsic::amdgcn_ps_live: {
4029 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4032 case Intrinsic::amdgcn_div_scale: {
4033 unsigned Dst0Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4034 unsigned Dst1Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
4035 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Dst0Size);
4036 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Dst1Size);
4038 unsigned SrcSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits();
4039 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4040 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4043 case Intrinsic::amdgcn_class: {
4044 Register Src0Reg = MI.getOperand(2).getReg();
4045 Register Src1Reg = MI.getOperand(3).getReg();
4046 unsigned Src0Size = MRI.getType(Src0Reg).getSizeInBits();
4047 unsigned Src1Size = MRI.getType(Src1Reg).getSizeInBits();
4048 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4049 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize);
4050 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size);
4051 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src1Size);
4054 case Intrinsic::amdgcn_icmp:
4055 case Intrinsic::amdgcn_fcmp: {
4056 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4057 // This is not VCCRegBank because this is not used in boolean contexts.
4058 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4059 unsigned OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4060 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
4061 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
4064 case Intrinsic::amdgcn_readlane: {
4065 // This must be an SGPR, but accept a VGPR.
4066 Register IdxReg = MI.getOperand(3).getReg();
4067 unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits();
4068 unsigned IdxBank = getRegBankID(IdxReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
4069 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4072 case Intrinsic::amdgcn_readfirstlane: {
4073 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4074 unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4075 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4076 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4079 case Intrinsic::amdgcn_writelane: {
4080 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4081 Register SrcReg = MI.getOperand(2).getReg();
4082 unsigned SrcSize = MRI.getType(SrcReg).getSizeInBits();
4083 unsigned SrcBank = getRegBankID(SrcReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
4084 Register IdxReg = MI.getOperand(3).getReg();
4085 unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits();
4086 unsigned IdxBank = getRegBankID(IdxReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
4087 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4089 // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
4091 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, SrcSize);
4092 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4093 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4096 case Intrinsic::amdgcn_if_break: {
4097 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4098 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4099 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4100 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4103 case Intrinsic::amdgcn_permlane16:
4104 case Intrinsic::amdgcn_permlanex16: {
4105 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4106 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4107 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4108 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4109 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4110 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4113 case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
4114 case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
4115 case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
4116 case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
4117 case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
4118 case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
4119 case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
4120 case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
4121 case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
4122 case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
4123 case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
4124 case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
4125 case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
4126 case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
4127 case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
4128 case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
4129 case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
4130 case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
4131 case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
4132 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16: {
4133 // Default for MAI intrinsics.
4134 // srcC can also be an immediate which can be folded later.
4135 // FIXME: Should we eventually add an alternative mapping with AGPR src
4138 // vdst, srcA, srcB, srcC
4139 OpdsMapping[0] = getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4140 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4141 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4142 OpdsMapping[4] = getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4145 case Intrinsic::amdgcn_interp_p1:
4146 case Intrinsic::amdgcn_interp_p2:
4147 case Intrinsic::amdgcn_interp_mov:
4148 case Intrinsic::amdgcn_interp_p1_f16:
4149 case Intrinsic::amdgcn_interp_p2_f16: {
4150 const int M0Idx = MI.getNumOperands() - 1;
4151 Register M0Reg = MI.getOperand(M0Idx).getReg();
4152 unsigned M0Bank = getRegBankID(M0Reg, MRI, *TRI, AMDGPU::SGPRRegBankID);
4153 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4155 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4156 for (int I = 2; I != M0Idx && MI.getOperand(I).isReg(); ++I)
4157 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4159 // Must be SGPR, but we must take whatever the original bank is and fix it
4161 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32);
4164 case Intrinsic::amdgcn_ballot: {
4165 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4166 unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4167 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4168 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize);
4174 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4175 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: {
4176 auto IntrID = MI.getIntrinsicID();
4177 const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(IntrID);
4178 assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic");
4179 // Non-images can have complications from operands that allow both SGPR
4180 // and VGPR. For now it's too complicated to figure out the final opcode
4181 // to derive the register bank from the MCInstrDesc.
4182 assert(RSrcIntrin->IsImage);
4183 return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg);
4185 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: {
4186 auto IntrID = MI.getIntrinsicID();
4188 case Intrinsic::amdgcn_s_getreg:
4189 case Intrinsic::amdgcn_s_memtime:
4190 case Intrinsic::amdgcn_s_memrealtime:
4191 case Intrinsic::amdgcn_s_get_waveid_in_workgroup: {
4192 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4193 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4196 case Intrinsic::amdgcn_ds_fadd:
4197 case Intrinsic::amdgcn_ds_fmin:
4198 case Intrinsic::amdgcn_ds_fmax:
4199 return getDefaultMappingAllVGPR(MI);
4200 case Intrinsic::amdgcn_ds_ordered_add:
4201 case Intrinsic::amdgcn_ds_ordered_swap: {
4202 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4203 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4204 unsigned M0Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
4205 AMDGPU::SGPRRegBankID);
4206 OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32);
4207 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4210 case Intrinsic::amdgcn_ds_append:
4211 case Intrinsic::amdgcn_ds_consume: {
4212 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4213 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4214 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4217 case Intrinsic::amdgcn_exp_compr:
4218 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4219 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4221 case Intrinsic::amdgcn_exp:
4222 // FIXME: Could we support packed types here?
4223 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4224 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4225 OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4226 OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4228 case Intrinsic::amdgcn_s_sendmsg:
4229 case Intrinsic::amdgcn_s_sendmsghalt: {
4230 // This must be an SGPR, but accept a VGPR.
4231 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
4232 AMDGPU::SGPRRegBankID);
4233 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4236 case Intrinsic::amdgcn_s_setreg: {
4237 // This must be an SGPR, but accept a VGPR.
4238 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
4239 AMDGPU::SGPRRegBankID);
4240 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4243 case Intrinsic::amdgcn_end_cf: {
4244 unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4245 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4248 case Intrinsic::amdgcn_else: {
4249 unsigned WaveSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4250 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4251 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
4252 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
4255 case Intrinsic::amdgcn_kill: {
4256 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4259 case Intrinsic::amdgcn_raw_buffer_load:
4260 case Intrinsic::amdgcn_raw_tbuffer_load: {
4261 // FIXME: Should make intrinsic ID the last operand of the instruction,
4262 // then this would be the same as store
4263 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4264 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4265 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4266 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4269 case Intrinsic::amdgcn_raw_buffer_store:
4270 case Intrinsic::amdgcn_raw_buffer_store_format:
4271 case Intrinsic::amdgcn_raw_tbuffer_store: {
4272 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4273 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4274 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4275 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4278 case Intrinsic::amdgcn_struct_buffer_load:
4279 case Intrinsic::amdgcn_struct_tbuffer_load: {
4280 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4281 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4282 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4283 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4284 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4287 case Intrinsic::amdgcn_struct_buffer_store:
4288 case Intrinsic::amdgcn_struct_tbuffer_store: {
4289 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4290 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4291 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4292 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4293 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4296 case Intrinsic::amdgcn_init_exec_from_input: {
4297 unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4298 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
4301 case Intrinsic::amdgcn_ds_gws_init:
4302 case Intrinsic::amdgcn_ds_gws_barrier:
4303 case Intrinsic::amdgcn_ds_gws_sema_br: {
4304 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4306 // This must be an SGPR, but accept a VGPR.
4307 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
4308 AMDGPU::SGPRRegBankID);
4309 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4312 case Intrinsic::amdgcn_ds_gws_sema_v:
4313 case Intrinsic::amdgcn_ds_gws_sema_p:
4314 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
4315 // This must be an SGPR, but accept a VGPR.
4316 unsigned Bank = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
4317 AMDGPU::SGPRRegBankID);
4318 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32);
4322 return getInvalidInstructionMapping();
4326 case AMDGPU::G_SELECT: {
4327 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4328 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
4329 AMDGPU::SGPRRegBankID);
4330 unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI,
4331 AMDGPU::SGPRRegBankID);
4332 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
4333 Op3Bank == AMDGPU::SGPRRegBankID;
4335 unsigned CondBankDefault = SGPRSrcs ?
4336 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4337 unsigned CondBank = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
4339 if (CondBank == AMDGPU::SGPRRegBankID)
4340 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4341 else if (CondBank == AMDGPU::VGPRRegBankID)
4342 CondBank = AMDGPU::VCCRegBankID;
4344 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ?
4345 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4347 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID);
4349 // TODO: Should report 32-bit for scalar condition type.
4351 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
4352 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1);
4353 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
4354 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
4356 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, Size);
4357 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1);
4358 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, Size);
4359 OpdsMapping[3] = AMDGPU::getValueMapping(Bank, Size);
4365 case AMDGPU::G_LOAD:
4366 case AMDGPU::G_ZEXTLOAD:
4367 case AMDGPU::G_SEXTLOAD:
4368 return getInstrMappingForLoad(MI);
4370 case AMDGPU::G_ATOMICRMW_XCHG:
4371 case AMDGPU::G_ATOMICRMW_ADD:
4372 case AMDGPU::G_ATOMICRMW_SUB:
4373 case AMDGPU::G_ATOMICRMW_AND:
4374 case AMDGPU::G_ATOMICRMW_OR:
4375 case AMDGPU::G_ATOMICRMW_XOR:
4376 case AMDGPU::G_ATOMICRMW_MAX:
4377 case AMDGPU::G_ATOMICRMW_MIN:
4378 case AMDGPU::G_ATOMICRMW_UMAX:
4379 case AMDGPU::G_ATOMICRMW_UMIN:
4380 case AMDGPU::G_ATOMICRMW_FADD:
4381 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG:
4382 case AMDGPU::G_AMDGPU_ATOMIC_INC:
4383 case AMDGPU::G_AMDGPU_ATOMIC_DEC: {
4384 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4385 OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg());
4386 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4389 case AMDGPU::G_ATOMIC_CMPXCHG: {
4390 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4391 OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg());
4392 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4393 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4396 case AMDGPU::G_BRCOND: {
4397 unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI,
4398 AMDGPU::SGPRRegBankID);
4399 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
4400 if (Bank != AMDGPU::SGPRRegBankID)
4401 Bank = AMDGPU::VCCRegBankID;
4403 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
4408 return getInstructionMapping(/*ID*/1, /*Cost*/1,
4409 getOperandsMapping(OpdsMapping),
4410 MI.getNumOperands());