1 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This pass creates bundles of SMEM and VMEM instructions forming memory
11 /// clauses if XNACK is enabled. Def operands of clauses are marked as early
12 /// clobber to make sure we will not override any source within a clause.
14 //===----------------------------------------------------------------------===//
17 #include "AMDGPUSubtarget.h"
18 #include "GCNRegPressure.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "SIInstrInfo.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "SIRegisterInfo.h"
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/CodeGen/LiveIntervals.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/InitializePasses.h"
30 #define DEBUG_TYPE "si-form-memory-clauses"
32 // Clauses longer then 15 instructions would overflow one of the counters
33 // and stall. They can stall even earlier if there are outstanding counters.
34 static cl::opt<unsigned>
35 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
36 cl::desc("Maximum length of a memory clause, instructions"));
40 class SIFormMemoryClauses : public MachineFunctionPass {
41 typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;
47 SIFormMemoryClauses() : MachineFunctionPass(ID) {
48 initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
51 bool runOnMachineFunction(MachineFunction &MF) override;
53 StringRef getPassName() const override {
54 return "SI Form memory clauses";
57 void getAnalysisUsage(AnalysisUsage &AU) const override {
58 AU.addRequired<LiveIntervals>();
60 MachineFunctionPass::getAnalysisUsage(AU);
64 template <typename Callable>
65 void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
67 bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
68 bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
69 void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
70 bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
71 GCNDownwardRPTracker &RPT);
73 const GCNSubtarget *ST;
74 const SIRegisterInfo *TRI;
75 const MachineRegisterInfo *MRI;
76 SIMachineFunctionInfo *MFI;
78 unsigned LastRecordedOccupancy;
83 } // End anonymous namespace.
85 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
86 "SI Form memory clauses", false, false)
87 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
88 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
89 "SI Form memory clauses", false, false)
92 char SIFormMemoryClauses::ID = 0;
94 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
96 FunctionPass *llvm::createSIFormMemoryClausesPass() {
97 return new SIFormMemoryClauses();
100 static bool isVMEMClauseInst(const MachineInstr &MI) {
101 return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
104 static bool isSMEMClauseInst(const MachineInstr &MI) {
105 return SIInstrInfo::isSMRD(MI);
108 // There no sense to create store clauses, they do not define anything,
109 // thus there is nothing to set early-clobber.
110 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
111 if (MI.isDebugValue() || MI.isBundled())
113 if (!MI.mayLoad() || MI.mayStore())
115 if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
116 AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
118 if (IsVMEMClause && !isVMEMClauseInst(MI))
120 if (!IsVMEMClause && !isSMEMClauseInst(MI))
122 // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
123 for (const MachineOperand &ResMO : MI.defs()) {
124 Register ResReg = ResMO.getReg();
125 for (const MachineOperand &MO : MI.uses()) {
126 if (!MO.isReg() || MO.isDef())
128 if (MO.getReg() == ResReg)
131 break; // Only check the first def.
136 static unsigned getMopState(const MachineOperand &MO) {
139 S |= RegState::Implicit;
143 S |= RegState::Undef;
146 if (MO.isEarlyClobber())
147 S |= RegState::EarlyClobber;
148 if (Register::isPhysicalRegister(MO.getReg()) && MO.isRenamable())
149 S |= RegState::Renamable;
153 template <typename Callable>
154 void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
155 Callable Func) const {
156 if (LaneMask.all() || Register::isPhysicalRegister(Reg) ||
157 LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) {
162 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
163 unsigned E = TRI->getNumSubRegIndices();
164 SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
165 for (unsigned Idx = 1; Idx < E; ++Idx) {
166 // Is this index even compatible with the given class?
167 if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
169 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
170 // Early exit if we found a perfect match.
171 if (SubRegMask == LaneMask) {
176 if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
179 CoveringSubregs.push_back(Idx);
182 llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
183 LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
184 LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
185 unsigned NA = MaskA.getNumLanes();
186 unsigned NB = MaskB.getNumLanes();
189 return MaskA.getHighestLane() > MaskB.getHighestLane();
192 for (unsigned Idx : CoveringSubregs) {
193 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
194 if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
198 LaneMask &= ~SubRegMask;
203 llvm_unreachable("Failed to find all subregs to cover lane mask");
206 // Returns false if there is a use of a def already in the map.
207 // In this case we must break the clause.
208 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
209 RegUse &Defs, RegUse &Uses) const {
210 // Check interference with defs.
211 for (const MachineOperand &MO : MI.operands()) {
212 // TODO: Prologue/Epilogue Insertion pass does not process bundled
220 Register Reg = MO.getReg();
222 // If it is tied we will need to write same register as we read.
226 RegUse &Map = MO.isDef() ? Uses : Defs;
227 auto Conflict = Map.find(Reg);
228 if (Conflict == Map.end())
231 if (Register::isPhysicalRegister(Reg))
234 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
235 if ((Conflict->second.second & Mask).any())
242 // Since all defs in the clause are early clobber we can run out of registers.
243 // Function returns false if pressure would hit the limit if instruction is
244 // bundled into a memory clause.
245 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
246 GCNDownwardRPTracker &RPT) {
247 // NB: skip advanceBeforeNext() call. Since all defs will be marked
248 // early-clobber they will all stay alive at least to the end of the
249 // clause. Therefor we should not decrease pressure even if load
250 // pointer becomes dead and could otherwise be reused for destination.
252 GCNRegPressure MaxPressure = RPT.moveMaxPressure();
253 unsigned Occupancy = MaxPressure.getOccupancy(*ST);
254 if (Occupancy >= MFI->getMinAllowedOccupancy() &&
255 MaxPressure.getVGPRNum() <= MaxVGPRs &&
256 MaxPressure.getSGPRNum() <= MaxSGPRs) {
257 LastRecordedOccupancy = Occupancy;
263 // Collect register defs and uses along with their lane masks and states.
264 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
265 RegUse &Defs, RegUse &Uses) const {
266 for (const MachineOperand &MO : MI.operands()) {
269 Register Reg = MO.getReg();
273 LaneBitmask Mask = Register::isVirtualRegister(Reg)
274 ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
275 : LaneBitmask::getAll();
276 RegUse &Map = MO.isDef() ? Defs : Uses;
278 auto Loc = Map.find(Reg);
279 unsigned State = getMopState(MO);
280 if (Loc == Map.end()) {
281 Map[Reg] = std::make_pair(State, Mask);
283 Loc->second.first |= State;
284 Loc->second.second |= Mask;
289 // Check register def/use conflicts, occupancy limits and collect def/use maps.
290 // Return true if instruction can be bundled with previous. It it cannot
291 // def/use maps are not updated.
292 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
293 RegUse &Defs, RegUse &Uses,
294 GCNDownwardRPTracker &RPT) {
295 if (!canBundle(MI, Defs, Uses))
298 if (!checkPressure(MI, RPT))
301 collectRegUses(MI, Defs, Uses);
305 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
306 if (skipFunction(MF.getFunction()))
309 ST = &MF.getSubtarget<GCNSubtarget>();
310 if (!ST->isXNACKEnabled())
313 const SIInstrInfo *TII = ST->getInstrInfo();
314 TRI = ST->getRegisterInfo();
315 MRI = &MF.getRegInfo();
316 MFI = MF.getInfo<SIMachineFunctionInfo>();
317 LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
318 SlotIndexes *Ind = LIS->getSlotIndexes();
319 bool Changed = false;
321 MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
322 MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
323 unsigned FuncMaxClause = AMDGPU::getIntegerAttribute(
324 MF.getFunction(), "amdgpu-max-memory-clause", MaxClause);
326 for (MachineBasicBlock &MBB : MF) {
327 MachineBasicBlock::instr_iterator Next;
328 for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
329 MachineInstr &MI = *I;
332 bool IsVMEM = isVMEMClauseInst(MI);
334 if (!isValidClauseInst(MI, IsVMEM))
338 GCNDownwardRPTracker RPT(*LIS);
341 if (!processRegUses(MI, Defs, Uses, RPT))
345 for ( ; Next != E && Length < FuncMaxClause; ++Next) {
346 if (!isValidClauseInst(*Next, IsVMEM))
349 // A load from pointer which was loaded inside the same bundle is an
350 // impossible clause because we will need to write and read the same
351 // register inside. In this case processRegUses will return false.
352 if (!processRegUses(*Next, Defs, Uses, RPT))
361 MFI->limitOccupancy(LastRecordedOccupancy);
363 auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
364 Ind->insertMachineInstrInMaps(*B);
366 for (auto BI = I; BI != Next; ++BI) {
367 BI->bundleWithPred();
368 Ind->removeSingleMachineInstrFromMaps(*BI);
370 for (MachineOperand &MO : BI->defs())
372 MO.setIsInternalRead(true);
375 for (auto &&R : Defs) {
376 forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
377 unsigned S = R.second.first | RegState::EarlyClobber;
379 S &= ~(RegState::Undef | RegState::Dead);
380 B.addDef(R.first, S, SubReg);
384 for (auto &&R : Uses) {
385 forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
386 B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
390 for (auto &&R : Defs) {
391 unsigned Reg = R.first;
393 if (Register::isPhysicalRegister(Reg))
395 LIS->removeInterval(Reg);
396 LIS->createAndComputeVirtRegInterval(Reg);
399 for (auto &&R : Uses) {
400 unsigned Reg = R.first;
401 if (Register::isPhysicalRegister(Reg))
403 LIS->removeInterval(Reg);
404 LIS->createAndComputeVirtRegInterval(Reg);