1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements hazard recognizers for scheduling on GCN processors.
12 //===----------------------------------------------------------------------===//
14 #include "GCNHazardRecognizer.h"
15 #include "AMDGPUSubtarget.h"
16 #include "SIInstrInfo.h"
17 #include "llvm/CodeGen/ScheduleDAG.h"
18 #include "llvm/Support/Debug.h"
22 //===----------------------------------------------------------------------===//
23 // Hazard Recoginizer Implementation
24 //===----------------------------------------------------------------------===//
26 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
27 CurrCycleInstr(nullptr),
29 ST(MF.getSubtarget<SISubtarget>()) {
33 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
34 EmitInstruction(SU->getInstr());
37 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
41 ScheduleHazardRecognizer::HazardType
42 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
43 MachineInstr *MI = SU->getInstr();
45 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
48 if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
51 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
57 unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
58 return PreEmitNoops(SU->getInstr());
61 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
62 if (SIInstrInfo::isSMRD(*MI))
63 return std::max(0, checkSMRDHazards(MI));
65 if (SIInstrInfo::isVMEM(*MI))
66 return std::max(0, checkVMEMHazards(MI));
68 if (SIInstrInfo::isDPP(*MI))
69 return std::max(0, checkDPPHazards(MI));
74 void GCNHazardRecognizer::EmitNoop() {
75 EmittedInstrs.push_front(nullptr);
78 void GCNHazardRecognizer::AdvanceCycle() {
80 // When the scheduler detects a stall, it will call AdvanceCycle() without
81 // emitting any instructions.
85 const SIInstrInfo *TII = ST.getInstrInfo();
86 unsigned NumWaitStates = TII->getNumWaitStates(*CurrCycleInstr);
88 // Keep track of emitted instructions
89 EmittedInstrs.push_front(CurrCycleInstr);
91 // Add a nullptr for each additional wait state after the first. Make sure
92 // not to add more than getMaxLookAhead() items to the list, since we
93 // truncate the list to that size right after this loop.
94 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
96 EmittedInstrs.push_front(nullptr);
99 // getMaxLookahead() is the largest number of wait states we will ever need
100 // to insert, so there is no point in keeping track of more than that many
102 EmittedInstrs.resize(getMaxLookAhead());
104 CurrCycleInstr = nullptr;
107 void GCNHazardRecognizer::RecedeCycle() {
108 llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
111 //===----------------------------------------------------------------------===//
113 //===----------------------------------------------------------------------===//
115 int GCNHazardRecognizer::getWaitStatesSinceDef(
116 unsigned Reg, function_ref<bool(MachineInstr *)> IsHazardDef) {
117 const SIRegisterInfo *TRI = ST.getRegisterInfo();
120 for (MachineInstr *MI : EmittedInstrs) {
122 if (!MI || !IsHazardDef(MI))
124 if (MI->modifiesRegister(Reg, TRI))
127 return std::numeric_limits<int>::max();
130 //===----------------------------------------------------------------------===//
131 // No-op Hazard Detection
132 //===----------------------------------------------------------------------===//
134 static void addRegsToSet(iterator_range<MachineInstr::const_mop_iterator> Ops,
135 std::set<unsigned> &Set) {
136 for (const MachineOperand &Op : Ops) {
138 Set.insert(Op.getReg());
142 int GCNHazardRecognizer::checkSMEMSoftClauseHazards(MachineInstr *SMEM) {
143 // SMEM soft clause are only present on VI+
144 if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS)
147 // A soft-clause is any group of consecutive SMEM instructions. The
148 // instructions in this group may return out of order and/or may be
149 // replayed (i.e. the same instruction issued more than once).
151 // In order to handle these situations correctly we need to make sure
152 // that when a clause has more than one instruction, no instruction in the
153 // clause writes to a register that is read another instruction in the clause
154 // (including itself). If we encounter this situaion, we need to break the
155 // clause by inserting a non SMEM instruction.
157 std::set<unsigned> ClauseDefs;
158 std::set<unsigned> ClauseUses;
160 for (MachineInstr *MI : EmittedInstrs) {
162 // When we hit a non-SMEM instruction then we have passed the start of the
163 // clause and we can stop.
164 if (!MI || !SIInstrInfo::isSMRD(*MI))
167 addRegsToSet(MI->defs(), ClauseDefs);
168 addRegsToSet(MI->uses(), ClauseUses);
171 if (ClauseDefs.empty())
174 // FIXME: When we support stores, we need to make sure not to put loads and
175 // stores in the same clause if they use the same address. For now, just
176 // start a new clause whenever we see a store.
177 if (SMEM->mayStore())
180 addRegsToSet(SMEM->defs(), ClauseDefs);
181 addRegsToSet(SMEM->uses(), ClauseUses);
183 std::vector<unsigned> Result(std::max(ClauseDefs.size(), ClauseUses.size()));
184 std::vector<unsigned>::iterator End;
186 End = std::set_intersection(ClauseDefs.begin(), ClauseDefs.end(),
187 ClauseUses.begin(), ClauseUses.end(), Result.begin());
189 // If the set of defs and uses intersect then we cannot add this instruction
190 // to the clause, so we have a hazard.
191 if (End != Result.begin())
197 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
198 const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
199 const SIInstrInfo *TII = ST.getInstrInfo();
200 int WaitStatesNeeded = 0;
202 WaitStatesNeeded = checkSMEMSoftClauseHazards(SMRD);
204 // This SMRD hazard only affects SI.
205 if (ST.getGeneration() != SISubtarget::SOUTHERN_ISLANDS)
206 return WaitStatesNeeded;
208 // A read of an SGPR by SMRD instruction requires 4 wait states when the
209 // SGPR was written by a VALU instruction.
210 int SmrdSgprWaitStates = 4;
211 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
213 for (const MachineOperand &Use : SMRD->uses()) {
216 int WaitStatesNeededForUse =
217 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
218 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
220 return WaitStatesNeeded;
223 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
224 const SIInstrInfo *TII = ST.getInstrInfo();
226 if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS)
229 const SIRegisterInfo &TRI = TII->getRegisterInfo();
231 // A read of an SGPR by a VMEM instruction requires 5 wait states when the
232 // SGPR was written by a VALU Instruction.
233 int VmemSgprWaitStates = 5;
234 int WaitStatesNeeded = 0;
235 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
237 for (const MachineOperand &Use : VMEM->uses()) {
238 if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
241 int WaitStatesNeededForUse =
242 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
243 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
245 return WaitStatesNeeded;
248 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
249 const SIRegisterInfo *TRI = ST.getRegisterInfo();
251 // Check for DPP VGPR read after VALU VGPR write.
252 int DppVgprWaitStates = 2;
253 int WaitStatesNeeded = 0;
255 for (const MachineOperand &Use : DPP->uses()) {
256 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
258 int WaitStatesNeededForUse =
259 DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg());
260 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
263 return WaitStatesNeeded;