1 //===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// Insert wait instructions for memory reads and writes.
12 /// Memory reads and writes are issued asynchronously, so we need to insert
13 /// S_WAITCNT instructions when we want to access any of their results or
14 /// overwrite any register that's used asynchronously.
16 /// TODO: This pass currently keeps one timeline per hardware counter. A more
17 /// finely-grained approach that keeps one timeline per event type could
18 /// sometimes get away with generating weaker s_waitcnt instructions. For
19 /// example, when both SMEM and LDS are in flight and we need to wait for
20 /// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21 /// but the pass will currently generate a conservative lgkmcnt(0) because
22 /// multiple event types are in flight.
24 //===----------------------------------------------------------------------===//
27 #include "AMDGPUSubtarget.h"
28 #include "SIDefines.h"
29 #include "SIInstrInfo.h"
30 #include "SIMachineFunctionInfo.h"
31 #include "SIRegisterInfo.h"
32 #include "Utils/AMDGPUBaseInfo.h"
33 #include "llvm/ADT/DenseMap.h"
34 #include "llvm/ADT/DenseSet.h"
35 #include "llvm/ADT/PostOrderIterator.h"
36 #include "llvm/ADT/STLExtras.h"
37 #include "llvm/ADT/SmallVector.h"
38 #include "llvm/CodeGen/MachineBasicBlock.h"
39 #include "llvm/CodeGen/MachineFunction.h"
40 #include "llvm/CodeGen/MachineFunctionPass.h"
41 #include "llvm/CodeGen/MachineInstr.h"
42 #include "llvm/CodeGen/MachineInstrBuilder.h"
43 #include "llvm/CodeGen/MachineMemOperand.h"
44 #include "llvm/CodeGen/MachineOperand.h"
45 #include "llvm/CodeGen/MachineRegisterInfo.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/Pass.h"
48 #include "llvm/Support/Debug.h"
49 #include "llvm/Support/DebugCounter.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/raw_ostream.h"
62 #define DEBUG_TYPE "si-insert-waitcnts"
64 DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE"-forceexp",
65 "Force emit s_waitcnt expcnt(0) instrs");
66 DEBUG_COUNTER(ForceLgkmCounter, DEBUG_TYPE"-forcelgkm",
67 "Force emit s_waitcnt lgkmcnt(0) instrs");
68 DEBUG_COUNTER(ForceVMCounter, DEBUG_TYPE"-forcevm",
69 "Force emit s_waitcnt vmcnt(0) instrs");
71 static cl::opt<bool> ForceEmitZeroFlag(
72 "amdgpu-waitcnt-forcezero",
73 cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
74 cl::init(false), cl::Hidden);
78 template <typename EnumT>
80 : public iterator_facade_base<enum_iterator<EnumT>,
81 std::forward_iterator_tag, const EnumT> {
84 enum_iterator() = default;
85 enum_iterator(EnumT Value) : Value(Value) {}
87 enum_iterator &operator++() {
88 Value = static_cast<EnumT>(Value + 1);
92 bool operator==(const enum_iterator &RHS) const { return Value == RHS.Value; }
94 EnumT operator*() const { return Value; }
97 // Class of object that encapsulates latest instruction counter score
98 // associated with the operand. Used for determining whether
99 // s_waitcnt instruction needs to be emited.
101 #define CNT_MASK(t) (1u << (t))
103 enum InstCounterType { VM_CNT = 0, LGKM_CNT, EXP_CNT, VS_CNT, NUM_INST_CNTS };
105 iterator_range<enum_iterator<InstCounterType>> inst_counter_types() {
106 return make_range(enum_iterator<InstCounterType>(VM_CNT),
107 enum_iterator<InstCounterType>(NUM_INST_CNTS));
110 using RegInterval = std::pair<signed, signed>;
129 VMEM_ACCESS, // vector-memory read & write
130 VMEM_READ_ACCESS, // vector-memory read
131 VMEM_WRITE_ACCESS,// vector-memory write
132 LDS_ACCESS, // lds read & write
133 GDS_ACCESS, // gds read & write
134 SQ_MESSAGE, // send message
135 SMEM_ACCESS, // scalar-memory read & write
136 EXP_GPR_LOCK, // export holding on its data src
137 GDS_GPR_LOCK, // GDS holding on its data and addr src
138 EXP_POS_ACCESS, // write to export position
139 EXP_PARAM_ACCESS, // write to export parameter
140 VMW_GPR_LOCK, // vector-memory write holding on its data src
144 static const uint32_t WaitEventMaskForInst[NUM_INST_CNTS] = {
145 (1 << VMEM_ACCESS) | (1 << VMEM_READ_ACCESS),
146 (1 << SMEM_ACCESS) | (1 << LDS_ACCESS) | (1 << GDS_ACCESS) |
148 (1 << EXP_GPR_LOCK) | (1 << GDS_GPR_LOCK) | (1 << VMW_GPR_LOCK) |
149 (1 << EXP_PARAM_ACCESS) | (1 << EXP_POS_ACCESS),
150 (1 << VMEM_WRITE_ACCESS)
154 // 0 .. SQ_MAX_PGM_VGPRS-1 real VGPRs
155 // SQ_MAX_PGM_VGPRS .. NUM_ALL_VGPRS-1 extra VGPR-like slots
156 // NUM_ALL_VGPRS .. NUM_ALL_VGPRS+SQ_MAX_PGM_SGPRS-1 real SGPRs
157 // We reserve a fixed number of VGPR slots in the scoring tables for
158 // special tokens like SCMEM_LDS (needed for buffer load to LDS).
159 enum RegisterMapping {
160 SQ_MAX_PGM_VGPRS = 256, // Maximum programmable VGPRs across all targets.
161 SQ_MAX_PGM_SGPRS = 256, // Maximum programmable SGPRs across all targets.
162 NUM_EXTRA_VGPRS = 1, // A reserved slot for DS.
163 EXTRA_VGPR_LDS = 0, // This is a placeholder the Shader algorithm uses.
164 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS, // Where SGPR starts.
167 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) {
170 Wait.VmCnt = std::min(Wait.VmCnt, Count);
173 Wait.ExpCnt = std::min(Wait.ExpCnt, Count);
176 Wait.LgkmCnt = std::min(Wait.LgkmCnt, Count);
179 Wait.VsCnt = std::min(Wait.VsCnt, Count);
182 llvm_unreachable("bad InstCounterType");
186 // This objects maintains the current score brackets of each wait counter, and
187 // a per-register scoreboard for each wait counter.
189 // We also maintain the latest score for every event type that can change the
190 // waitcnt in order to know if there are multiple types of events within
191 // the brackets. When multiple types of event happen in the bracket,
192 // wait count may get decreased out of order, therefore we need to put in
193 // "s_waitcnt 0" before use.
194 class WaitcntBrackets {
196 WaitcntBrackets(const GCNSubtarget *SubTarget) : ST(SubTarget) {
197 for (auto T : inst_counter_types())
198 memset(VgprScores[T], 0, sizeof(VgprScores[T]));
201 static uint32_t getWaitCountMax(InstCounterType T) {
204 return HardwareLimits.VmcntMax;
206 return HardwareLimits.LgkmcntMax;
208 return HardwareLimits.ExpcntMax;
210 return HardwareLimits.VscntMax;
217 uint32_t getScoreLB(InstCounterType T) const {
218 assert(T < NUM_INST_CNTS);
219 if (T >= NUM_INST_CNTS)
224 uint32_t getScoreUB(InstCounterType T) const {
225 assert(T < NUM_INST_CNTS);
226 if (T >= NUM_INST_CNTS)
231 // Mapping from event to counter.
232 InstCounterType eventCounter(WaitEventType E) {
233 if (WaitEventMaskForInst[VM_CNT] & (1 << E))
235 if (WaitEventMaskForInst[LGKM_CNT] & (1 << E))
237 if (WaitEventMaskForInst[VS_CNT] & (1 << E))
239 assert(WaitEventMaskForInst[EXP_CNT] & (1 << E));
243 uint32_t getRegScore(int GprNo, InstCounterType T) {
244 if (GprNo < NUM_ALL_VGPRS) {
245 return VgprScores[T][GprNo];
247 assert(T == LGKM_CNT);
248 return SgprScores[GprNo - NUM_ALL_VGPRS];
252 memset(ScoreLBs, 0, sizeof(ScoreLBs));
253 memset(ScoreUBs, 0, sizeof(ScoreUBs));
255 memset(MixedPendingEvents, 0, sizeof(MixedPendingEvents));
256 for (auto T : inst_counter_types())
257 memset(VgprScores[T], 0, sizeof(VgprScores[T]));
258 memset(SgprScores, 0, sizeof(SgprScores));
261 bool merge(const WaitcntBrackets &Other);
263 RegInterval getRegInterval(const MachineInstr *MI, const SIInstrInfo *TII,
264 const MachineRegisterInfo *MRI,
265 const SIRegisterInfo *TRI, unsigned OpNo,
268 int32_t getMaxVGPR() const { return VgprUB; }
269 int32_t getMaxSGPR() const { return SgprUB; }
271 bool counterOutOfOrder(InstCounterType T) const;
272 bool simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
273 bool simplifyWaitcnt(InstCounterType T, unsigned &Count) const;
274 void determineWait(InstCounterType T, uint32_t ScoreToWait,
275 AMDGPU::Waitcnt &Wait) const;
276 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
277 void applyWaitcnt(InstCounterType T, unsigned Count);
278 void updateByEvent(const SIInstrInfo *TII, const SIRegisterInfo *TRI,
279 const MachineRegisterInfo *MRI, WaitEventType E,
282 bool hasPending() const { return PendingEvents != 0; }
283 bool hasPendingEvent(WaitEventType E) const {
284 return PendingEvents & (1 << E);
287 bool hasPendingFlat() const {
288 return ((LastFlat[LGKM_CNT] > ScoreLBs[LGKM_CNT] &&
289 LastFlat[LGKM_CNT] <= ScoreUBs[LGKM_CNT]) ||
290 (LastFlat[VM_CNT] > ScoreLBs[VM_CNT] &&
291 LastFlat[VM_CNT] <= ScoreUBs[VM_CNT]));
294 void setPendingFlat() {
295 LastFlat[VM_CNT] = ScoreUBs[VM_CNT];
296 LastFlat[LGKM_CNT] = ScoreUBs[LGKM_CNT];
299 void print(raw_ostream &);
300 void dump() { print(dbgs()); }
309 static bool mergeScore(const MergeInfo &M, uint32_t &Score,
310 uint32_t OtherScore);
312 void setScoreLB(InstCounterType T, uint32_t Val) {
313 assert(T < NUM_INST_CNTS);
314 if (T >= NUM_INST_CNTS)
319 void setScoreUB(InstCounterType T, uint32_t Val) {
320 assert(T < NUM_INST_CNTS);
321 if (T >= NUM_INST_CNTS)
325 uint32_t UB = ScoreUBs[T] - getWaitCountMax(EXP_CNT);
326 if (ScoreLBs[T] < UB && UB < ScoreUBs[T])
331 void setRegScore(int GprNo, InstCounterType T, uint32_t Val) {
332 if (GprNo < NUM_ALL_VGPRS) {
333 if (GprNo > VgprUB) {
336 VgprScores[T][GprNo] = Val;
338 assert(T == LGKM_CNT);
339 if (GprNo - NUM_ALL_VGPRS > SgprUB) {
340 SgprUB = GprNo - NUM_ALL_VGPRS;
342 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
346 void setExpScore(const MachineInstr *MI, const SIInstrInfo *TII,
347 const SIRegisterInfo *TRI, const MachineRegisterInfo *MRI,
348 unsigned OpNo, uint32_t Val);
350 const GCNSubtarget *ST = nullptr;
351 uint32_t ScoreLBs[NUM_INST_CNTS] = {0};
352 uint32_t ScoreUBs[NUM_INST_CNTS] = {0};
353 uint32_t PendingEvents = 0;
354 bool MixedPendingEvents[NUM_INST_CNTS] = {false};
355 // Remember the last flat memory operation.
356 uint32_t LastFlat[NUM_INST_CNTS] = {0};
357 // wait_cnt scores for every vgpr.
358 // Keep track of the VgprUB and SgprUB to make merge at join efficient.
361 uint32_t VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS];
362 // Wait cnt scores for every sgpr, only lgkmcnt is relevant.
363 uint32_t SgprScores[SQ_MAX_PGM_SGPRS] = {0};
366 class SIInsertWaitcnts : public MachineFunctionPass {
368 const GCNSubtarget *ST = nullptr;
369 const SIInstrInfo *TII = nullptr;
370 const SIRegisterInfo *TRI = nullptr;
371 const MachineRegisterInfo *MRI = nullptr;
372 AMDGPU::IsaVersion IV;
374 DenseSet<MachineInstr *> TrackedWaitcntSet;
375 DenseSet<MachineInstr *> VCCZBugHandledSet;
378 MachineBasicBlock *MBB;
379 std::unique_ptr<WaitcntBrackets> Incoming;
382 explicit BlockInfo(MachineBasicBlock *MBB) : MBB(MBB) {}
385 std::vector<BlockInfo> BlockInfos; // by reverse post-order traversal index
386 DenseMap<MachineBasicBlock *, unsigned> RpotIdxMap;
388 // ForceEmitZeroWaitcnts: force all waitcnts insts to be s_waitcnt 0
389 // because of amdgpu-waitcnt-forcezero flag
390 bool ForceEmitZeroWaitcnts;
391 bool ForceEmitWaitcnt[NUM_INST_CNTS];
396 SIInsertWaitcnts() : MachineFunctionPass(ID) {
397 (void)ForceExpCounter;
398 (void)ForceLgkmCounter;
399 (void)ForceVMCounter;
402 bool runOnMachineFunction(MachineFunction &MF) override;
404 StringRef getPassName() const override {
405 return "SI insert wait instructions";
408 void getAnalysisUsage(AnalysisUsage &AU) const override {
409 AU.setPreservesCFG();
410 MachineFunctionPass::getAnalysisUsage(AU);
413 bool isForceEmitWaitcnt() const {
414 for (auto T : inst_counter_types())
415 if (ForceEmitWaitcnt[T])
420 void setForceEmitWaitcnt() {
421 // For non-debug builds, ForceEmitWaitcnt has been initialized to false;
422 // For debug builds, get the debug counter info and adjust if need be
424 if (DebugCounter::isCounterSet(ForceExpCounter) &&
425 DebugCounter::shouldExecute(ForceExpCounter)) {
426 ForceEmitWaitcnt[EXP_CNT] = true;
428 ForceEmitWaitcnt[EXP_CNT] = false;
431 if (DebugCounter::isCounterSet(ForceLgkmCounter) &&
432 DebugCounter::shouldExecute(ForceLgkmCounter)) {
433 ForceEmitWaitcnt[LGKM_CNT] = true;
435 ForceEmitWaitcnt[LGKM_CNT] = false;
438 if (DebugCounter::isCounterSet(ForceVMCounter) &&
439 DebugCounter::shouldExecute(ForceVMCounter)) {
440 ForceEmitWaitcnt[VM_CNT] = true;
442 ForceEmitWaitcnt[VM_CNT] = false;
447 bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
448 bool generateWaitcntInstBefore(MachineInstr &MI,
449 WaitcntBrackets &ScoreBrackets,
450 MachineInstr *OldWaitcntInstr);
451 void updateEventWaitcntAfter(MachineInstr &Inst,
452 WaitcntBrackets *ScoreBrackets);
453 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
454 WaitcntBrackets &ScoreBrackets);
457 } // end anonymous namespace
459 RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI,
460 const SIInstrInfo *TII,
461 const MachineRegisterInfo *MRI,
462 const SIRegisterInfo *TRI,
463 unsigned OpNo, bool Def) const {
464 const MachineOperand &Op = MI->getOperand(OpNo);
465 if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg()) ||
466 (Def && !Op.isDef()) || TRI->isAGPR(*MRI, Op.getReg()))
469 // A use via a PW operand does not need a waitcnt.
470 // A partial write is not a WAW.
471 assert(!Op.getSubReg() || !Op.isUndef());
474 const MachineRegisterInfo &MRIA = *MRI;
476 unsigned Reg = TRI->getEncodingValue(Op.getReg());
478 if (TRI->isVGPR(MRIA, Op.getReg())) {
479 assert(Reg >= RegisterEncoding.VGPR0 && Reg <= RegisterEncoding.VGPRL);
480 Result.first = Reg - RegisterEncoding.VGPR0;
481 assert(Result.first >= 0 && Result.first < SQ_MAX_PGM_VGPRS);
482 } else if (TRI->isSGPRReg(MRIA, Op.getReg())) {
483 assert(Reg >= RegisterEncoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
484 Result.first = Reg - RegisterEncoding.SGPR0 + NUM_ALL_VGPRS;
485 assert(Result.first >= NUM_ALL_VGPRS &&
486 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
489 // else if (TRI->isTTMP(MRIA, Reg.getReg())) ...
493 const MachineInstr &MIA = *MI;
494 const TargetRegisterClass *RC = TII->getOpRegClass(MIA, OpNo);
495 unsigned Size = TRI->getRegSizeInBits(*RC);
496 Result.second = Result.first + (Size / 32);
501 void WaitcntBrackets::setExpScore(const MachineInstr *MI,
502 const SIInstrInfo *TII,
503 const SIRegisterInfo *TRI,
504 const MachineRegisterInfo *MRI, unsigned OpNo,
506 RegInterval Interval = getRegInterval(MI, TII, MRI, TRI, OpNo, false);
508 const MachineOperand &Opnd = MI->getOperand(OpNo);
509 assert(TRI->isVGPR(*MRI, Opnd.getReg()));
511 for (signed RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
512 setRegScore(RegNo, EXP_CNT, Val);
516 void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
517 const SIRegisterInfo *TRI,
518 const MachineRegisterInfo *MRI,
519 WaitEventType E, MachineInstr &Inst) {
520 const MachineRegisterInfo &MRIA = *MRI;
521 InstCounterType T = eventCounter(E);
522 uint32_t CurrScore = getScoreUB(T) + 1;
524 report_fatal_error("InsertWaitcnt score wraparound");
525 // PendingEvents and ScoreUB need to be update regardless if this event
526 // changes the score of a register or not.
527 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
528 if (!hasPendingEvent(E)) {
529 if (PendingEvents & WaitEventMaskForInst[T])
530 MixedPendingEvents[T] = true;
531 PendingEvents |= 1 << E;
533 setScoreUB(T, CurrScore);
536 // Put score on the source vgprs. If this is a store, just use those
537 // specific register(s).
538 if (TII->isDS(Inst) && (Inst.mayStore() || Inst.mayLoad())) {
540 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr);
541 // All GDS operations must protect their address register (same as
543 if (AddrOpIdx != -1) {
544 setExpScore(&Inst, TII, TRI, MRI, AddrOpIdx, CurrScore);
547 if (Inst.mayStore()) {
548 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
549 AMDGPU::OpName::data0) != -1) {
551 &Inst, TII, TRI, MRI,
552 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0),
555 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
556 AMDGPU::OpName::data1) != -1) {
557 setExpScore(&Inst, TII, TRI, MRI,
558 AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
559 AMDGPU::OpName::data1),
562 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1 &&
563 Inst.getOpcode() != AMDGPU::DS_GWS_INIT &&
564 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_V &&
565 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_BR &&
566 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_P &&
567 Inst.getOpcode() != AMDGPU::DS_GWS_BARRIER &&
568 Inst.getOpcode() != AMDGPU::DS_APPEND &&
569 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
570 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
571 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
572 const MachineOperand &Op = Inst.getOperand(I);
573 if (Op.isReg() && !Op.isDef() && TRI->isVGPR(MRIA, Op.getReg())) {
574 setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);
578 } else if (TII->isFLAT(Inst)) {
579 if (Inst.mayStore()) {
581 &Inst, TII, TRI, MRI,
582 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
584 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1) {
586 &Inst, TII, TRI, MRI,
587 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
590 } else if (TII->isMIMG(Inst)) {
591 if (Inst.mayStore()) {
592 setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
593 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1) {
595 &Inst, TII, TRI, MRI,
596 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
599 } else if (TII->isMTBUF(Inst)) {
600 if (Inst.mayStore()) {
601 setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
603 } else if (TII->isMUBUF(Inst)) {
604 if (Inst.mayStore()) {
605 setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
606 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1) {
608 &Inst, TII, TRI, MRI,
609 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
613 if (TII->isEXP(Inst)) {
614 // For export the destination registers are really temps that
615 // can be used as the actual source after export patching, so
616 // we need to treat them like sources and set the EXP_CNT
618 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
619 MachineOperand &DefMO = Inst.getOperand(I);
620 if (DefMO.isReg() && DefMO.isDef() &&
621 TRI->isVGPR(MRIA, DefMO.getReg())) {
622 setRegScore(TRI->getEncodingValue(DefMO.getReg()), EXP_CNT,
627 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
628 MachineOperand &MO = Inst.getOperand(I);
629 if (MO.isReg() && !MO.isDef() && TRI->isVGPR(MRIA, MO.getReg())) {
630 setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);
634 #if 0 // TODO: check if this is handled by MUBUF code above.
635 } else if (Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORD ||
636 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 ||
637 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) {
638 MachineOperand *MO = TII->getNamedOperand(Inst, AMDGPU::OpName::data);
639 unsigned OpNo;//TODO: find the OpNo for this operand;
640 RegInterval Interval = getRegInterval(&Inst, TII, MRI, TRI, OpNo, false);
641 for (signed RegNo = Interval.first; RegNo < Interval.second;
643 setRegScore(RegNo + NUM_ALL_VGPRS, t, CurrScore);
647 // Match the score to the destination registers.
648 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
649 RegInterval Interval = getRegInterval(&Inst, TII, MRI, TRI, I, true);
650 if (T == VM_CNT && Interval.first >= NUM_ALL_VGPRS)
652 for (signed RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
653 setRegScore(RegNo, T, CurrScore);
656 if (TII->isDS(Inst) && Inst.mayStore()) {
657 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS, T, CurrScore);
662 void WaitcntBrackets::print(raw_ostream &OS) {
664 for (auto T : inst_counter_types()) {
665 uint32_t LB = getScoreLB(T);
666 uint32_t UB = getScoreUB(T);
670 OS << " VM_CNT(" << UB - LB << "): ";
673 OS << " LGKM_CNT(" << UB - LB << "): ";
676 OS << " EXP_CNT(" << UB - LB << "): ";
679 OS << " VS_CNT(" << UB - LB << "): ";
682 OS << " UNKNOWN(" << UB - LB << "): ";
687 // Print vgpr scores.
688 for (int J = 0; J <= getMaxVGPR(); J++) {
689 uint32_t RegScore = getRegScore(J, T);
692 uint32_t RelScore = RegScore - LB - 1;
693 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
694 OS << RelScore << ":v" << J << " ";
696 OS << RelScore << ":ds ";
699 // Also need to print sgpr scores for lgkm_cnt.
701 for (int J = 0; J <= getMaxSGPR(); J++) {
702 uint32_t RegScore = getRegScore(J + NUM_ALL_VGPRS, LGKM_CNT);
705 uint32_t RelScore = RegScore - LB - 1;
706 OS << RelScore << ":s" << J << " ";
715 /// Simplify the waitcnt, in the sense of removing redundant counts, and return
716 /// whether a waitcnt instruction is needed at all.
717 bool WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
718 return simplifyWaitcnt(VM_CNT, Wait.VmCnt) |
719 simplifyWaitcnt(EXP_CNT, Wait.ExpCnt) |
720 simplifyWaitcnt(LGKM_CNT, Wait.LgkmCnt) |
721 simplifyWaitcnt(VS_CNT, Wait.VsCnt);
724 bool WaitcntBrackets::simplifyWaitcnt(InstCounterType T,
725 unsigned &Count) const {
726 const uint32_t LB = getScoreLB(T);
727 const uint32_t UB = getScoreUB(T);
728 if (Count < UB && UB - Count > LB)
735 void WaitcntBrackets::determineWait(InstCounterType T, uint32_t ScoreToWait,
736 AMDGPU::Waitcnt &Wait) const {
737 // If the score of src_operand falls within the bracket, we need an
738 // s_waitcnt instruction.
739 const uint32_t LB = getScoreLB(T);
740 const uint32_t UB = getScoreUB(T);
741 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
742 if ((T == VM_CNT || T == LGKM_CNT) &&
744 !ST->hasFlatLgkmVMemCountInOrder()) {
745 // If there is a pending FLAT operation, and this is a VMem or LGKM
746 // waitcnt and the target can report early completion, then we need
747 // to force a waitcnt 0.
749 } else if (counterOutOfOrder(T)) {
750 // Counter can get decremented out-of-order when there
751 // are multiple types event in the bracket. Also emit an s_wait counter
752 // with a conservative value of 0 for the counter.
755 addWait(Wait, T, UB - ScoreToWait);
760 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
761 applyWaitcnt(VM_CNT, Wait.VmCnt);
762 applyWaitcnt(EXP_CNT, Wait.ExpCnt);
763 applyWaitcnt(LGKM_CNT, Wait.LgkmCnt);
764 applyWaitcnt(VS_CNT, Wait.VsCnt);
767 void WaitcntBrackets::applyWaitcnt(InstCounterType T, unsigned Count) {
768 const uint32_t UB = getScoreUB(T);
772 if (counterOutOfOrder(T))
774 setScoreLB(T, std::max(getScoreLB(T), UB - Count));
777 MixedPendingEvents[T] = false;
778 PendingEvents &= ~WaitEventMaskForInst[T];
782 // Where there are multiple types of event in the bracket of a counter,
783 // the decrement may go out of order.
784 bool WaitcntBrackets::counterOutOfOrder(InstCounterType T) const {
785 // Scalar memory read always can go out of order.
786 if (T == LGKM_CNT && hasPendingEvent(SMEM_ACCESS))
788 return MixedPendingEvents[T];
791 INITIALIZE_PASS_BEGIN(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false,
793 INITIALIZE_PASS_END(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false,
796 char SIInsertWaitcnts::ID = 0;
798 char &llvm::SIInsertWaitcntsID = SIInsertWaitcnts::ID;
800 FunctionPass *llvm::createSIInsertWaitcntsPass() {
801 return new SIInsertWaitcnts();
804 static bool readsVCCZ(const MachineInstr &MI) {
805 unsigned Opc = MI.getOpcode();
806 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
807 !MI.getOperand(1).isUndef();
810 /// \returns true if the callee inserts an s_waitcnt 0 on function entry.
811 static bool callWaitsOnFunctionEntry(const MachineInstr &MI) {
812 // Currently all conventions wait, but this may not always be the case.
814 // TODO: If IPRA is enabled, and the callee is isSafeForNoCSROpt, it may make
815 // senses to omit the wait and do it in the caller.
819 /// \returns true if the callee is expected to wait for any outstanding waits
820 /// before returning.
821 static bool callWaitsOnFunctionReturn(const MachineInstr &MI) {
825 /// Generate s_waitcnt instruction to be placed before cur_Inst.
826 /// Instructions of a given type are returned in order,
827 /// but instructions of different types can complete out of order.
828 /// We rely on this in-order completion
829 /// and simply assign a score to the memory access instructions.
830 /// We keep track of the active "score bracket" to determine
831 /// if an access of a memory read requires an s_waitcnt
832 /// and if so what the value of each counter is.
833 /// The "score bracket" is bound by the lower bound and upper bound
834 /// scores (*_score_LB and *_score_ub respectively).
835 bool SIInsertWaitcnts::generateWaitcntInstBefore(
836 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
837 MachineInstr *OldWaitcntInstr) {
838 setForceEmitWaitcnt();
839 bool IsForceEmitWaitcnt = isForceEmitWaitcnt();
841 if (MI.isDebugInstr())
844 AMDGPU::Waitcnt Wait;
846 // See if this instruction has a forced S_WAITCNT VM.
847 // TODO: Handle other cases of NeedsWaitcntVmBefore()
848 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
849 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
850 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
851 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
852 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
856 // All waits must be resolved at call return.
857 // NOTE: this could be improved with knowledge of all call sites or
858 // with knowledge of the called routines.
859 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
860 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
861 (MI.isReturn() && MI.isCall() && !callWaitsOnFunctionEntry(MI))) {
862 Wait = Wait.combined(AMDGPU::Waitcnt::allZero(IV));
864 // Resolve vm waits before gs-done.
865 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG ||
866 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
867 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_) ==
868 AMDGPU::SendMsg::ID_GS_DONE)) {
871 #if 0 // TODO: the following blocks of logic when we have fence.
872 else if (MI.getOpcode() == SC_FENCE) {
873 const unsigned int group_size =
874 context->shader_info->GetMaxThreadGroupSize();
875 // group_size == 0 means thread group size is unknown at compile time
876 const bool group_is_multi_wave =
877 (group_size == 0 || group_size > target_info->GetWaveFrontSize());
878 const bool fence_is_global = !((SCInstInternalMisc*)Inst)->IsGroupFence();
880 for (unsigned int i = 0; i < Inst->NumSrcOperands(); i++) {
881 SCRegType src_type = Inst->GetSrcType(i);
884 if (group_is_multi_wave ||
885 context->OptFlagIsOn(OPT_R1100_LDSMEM_FENCE_CHICKEN_BIT)) {
886 EmitWaitcnt |= ScoreBrackets->updateByWait(LGKM_CNT,
887 ScoreBrackets->getScoreUB(LGKM_CNT));
888 // LDS may have to wait for VM_CNT after buffer load to LDS
889 if (target_info->HasBufferLoadToLDS()) {
890 EmitWaitcnt |= ScoreBrackets->updateByWait(VM_CNT,
891 ScoreBrackets->getScoreUB(VM_CNT));
897 if (group_is_multi_wave || fence_is_global) {
898 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
899 ScoreBrackets->getScoreUB(EXP_CNT));
900 EmitWaitcnt |= ScoreBrackets->updateByWait(LGKM_CNT,
901 ScoreBrackets->getScoreUB(LGKM_CNT));
909 if (group_is_multi_wave || fence_is_global) {
910 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
911 ScoreBrackets->getScoreUB(EXP_CNT));
912 EmitWaitcnt |= ScoreBrackets->updateByWait(VM_CNT,
913 ScoreBrackets->getScoreUB(VM_CNT));
925 // Export & GDS instructions do not read the EXEC mask until after the export
926 // is granted (which can occur well after the instruction is issued).
927 // The shader program must flush all EXP operations on the export-count
928 // before overwriting the EXEC mask.
930 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) {
931 // Export and GDS are tracked individually, either may trigger a waitcnt
933 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
934 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
935 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
936 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
941 if (MI.isCall() && callWaitsOnFunctionEntry(MI)) {
942 // Don't bother waiting on anything except the call address. The function
943 // is going to insert a wait on everything in its prolog. This still needs
944 // to be careful if the call target is a load (e.g. a GOT load).
945 Wait = AMDGPU::Waitcnt();
948 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
949 RegInterval Interval = ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI,
950 CallAddrOpIdx, false);
951 for (signed RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
952 ScoreBrackets.determineWait(
953 LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
956 // FIXME: Should not be relying on memoperands.
957 // Look at the source operands of every instruction to see if
958 // any of them results from a previous memory operation that affects
959 // its current usage. If so, an s_waitcnt instruction needs to be
961 // If the source operand was defined by a load, add the s_waitcnt
963 for (const MachineMemOperand *Memop : MI.memoperands()) {
964 unsigned AS = Memop->getAddrSpace();
965 if (AS != AMDGPUAS::LOCAL_ADDRESS)
967 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
968 // VM_CNT is only relevant to vgpr or LDS.
969 ScoreBrackets.determineWait(
970 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
973 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
974 const MachineOperand &Op = MI.getOperand(I);
975 const MachineRegisterInfo &MRIA = *MRI;
976 RegInterval Interval =
977 ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, I, false);
978 for (signed RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
979 if (TRI->isVGPR(MRIA, Op.getReg())) {
980 // VM_CNT is only relevant to vgpr or LDS.
981 ScoreBrackets.determineWait(
982 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
984 ScoreBrackets.determineWait(
985 LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
988 // End of for loop that looks at all source operands to decide vm_wait_cnt
991 // Two cases are handled for destination operands:
992 // 1) If the destination operand was defined by a load, add the s_waitcnt
993 // instruction to guarantee the right WAW order.
994 // 2) If a destination operand that was used by a recent export/store ins,
995 // add s_waitcnt on exp_cnt to guarantee the WAR order.
997 // FIXME: Should not be relying on memoperands.
998 for (const MachineMemOperand *Memop : MI.memoperands()) {
999 unsigned AS = Memop->getAddrSpace();
1000 if (AS != AMDGPUAS::LOCAL_ADDRESS)
1002 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1003 ScoreBrackets.determineWait(
1004 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
1005 ScoreBrackets.determineWait(
1006 EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait);
1009 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
1010 MachineOperand &Def = MI.getOperand(I);
1011 const MachineRegisterInfo &MRIA = *MRI;
1012 RegInterval Interval =
1013 ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, I, true);
1014 for (signed RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
1015 if (TRI->isVGPR(MRIA, Def.getReg())) {
1016 ScoreBrackets.determineWait(
1017 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
1018 ScoreBrackets.determineWait(
1019 EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait);
1021 ScoreBrackets.determineWait(
1022 LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
1024 } // End of for loop that looks at all dest operands.
1028 // Check to see if this is an S_BARRIER, and if an implicit S_WAITCNT 0
1029 // occurs before the instruction. Doing it here prevents any additional
1030 // S_WAITCNTs from being emitted if the instruction was marked as
1031 // requiring a WAITCNT beforehand.
1032 if (MI.getOpcode() == AMDGPU::S_BARRIER &&
1033 !ST->hasAutoWaitcntBeforeBarrier()) {
1034 Wait = Wait.combined(AMDGPU::Waitcnt::allZero(IV));
1037 // TODO: Remove this work-around, enable the assert for Bug 457939
1038 // after fixing the scheduler. Also, the Shader Compiler code is
1039 // independent of target.
1040 if (readsVCCZ(MI) && ST->hasReadVCCZBug()) {
1041 if (ScoreBrackets.getScoreLB(LGKM_CNT) <
1042 ScoreBrackets.getScoreUB(LGKM_CNT) &&
1043 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1048 // Early-out if no wait is indicated.
1049 if (!ScoreBrackets.simplifyWaitcnt(Wait) && !IsForceEmitWaitcnt) {
1050 bool Modified = false;
1051 if (OldWaitcntInstr) {
1052 for (auto II = OldWaitcntInstr->getIterator(), NextI = std::next(II);
1053 &*II != &MI; II = NextI, ++NextI) {
1054 if (II->isDebugInstr())
1057 if (TrackedWaitcntSet.count(&*II)) {
1058 TrackedWaitcntSet.erase(&*II);
1059 II->eraseFromParent();
1061 } else if (II->getOpcode() == AMDGPU::S_WAITCNT) {
1062 int64_t Imm = II->getOperand(0).getImm();
1063 ScoreBrackets.applyWaitcnt(AMDGPU::decodeWaitcnt(IV, Imm));
1065 assert(II->getOpcode() == AMDGPU::S_WAITCNT_VSCNT);
1066 assert(II->getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1067 ScoreBrackets.applyWaitcnt(
1068 AMDGPU::Waitcnt(0, 0, 0, II->getOperand(1).getImm()));
1075 if (ForceEmitZeroWaitcnts)
1076 Wait = AMDGPU::Waitcnt::allZero(IV);
1078 if (ForceEmitWaitcnt[VM_CNT])
1080 if (ForceEmitWaitcnt[EXP_CNT])
1082 if (ForceEmitWaitcnt[LGKM_CNT])
1084 if (ForceEmitWaitcnt[VS_CNT])
1087 ScoreBrackets.applyWaitcnt(Wait);
1089 AMDGPU::Waitcnt OldWait;
1090 bool Modified = false;
1092 if (OldWaitcntInstr) {
1093 for (auto II = OldWaitcntInstr->getIterator(), NextI = std::next(II);
1094 &*II != &MI; II = NextI, NextI++) {
1095 if (II->isDebugInstr())
1098 if (II->getOpcode() == AMDGPU::S_WAITCNT) {
1099 unsigned IEnc = II->getOperand(0).getImm();
1100 AMDGPU::Waitcnt IWait = AMDGPU::decodeWaitcnt(IV, IEnc);
1101 OldWait = OldWait.combined(IWait);
1102 if (!TrackedWaitcntSet.count(&*II))
1103 Wait = Wait.combined(IWait);
1104 unsigned NewEnc = AMDGPU::encodeWaitcnt(IV, Wait);
1105 if (IEnc != NewEnc) {
1106 II->getOperand(0).setImm(NewEnc);
1113 assert(II->getOpcode() == AMDGPU::S_WAITCNT_VSCNT);
1114 assert(II->getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1116 unsigned ICnt = II->getOperand(1).getImm();
1117 OldWait.VsCnt = std::min(OldWait.VsCnt, ICnt);
1118 if (!TrackedWaitcntSet.count(&*II))
1119 Wait.VsCnt = std::min(Wait.VsCnt, ICnt);
1120 if (Wait.VsCnt != ICnt) {
1121 II->getOperand(1).setImm(Wait.VsCnt);
1127 LLVM_DEBUG(dbgs() << "updateWaitcntInBlock\n"
1128 << "Old Instr: " << MI << '\n'
1129 << "New Instr: " << *II << '\n');
1131 if (!Wait.hasWait())
1136 if (Wait.VmCnt != ~0u || Wait.LgkmCnt != ~0u || Wait.ExpCnt != ~0u) {
1137 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait);
1138 auto SWaitInst = BuildMI(*MI.getParent(), MI.getIterator(),
1139 MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
1141 TrackedWaitcntSet.insert(SWaitInst);
1144 LLVM_DEBUG(dbgs() << "insertWaitcntInBlock\n"
1145 << "Old Instr: " << MI << '\n'
1146 << "New Instr: " << *SWaitInst << '\n');
1149 if (Wait.VsCnt != ~0u) {
1150 assert(ST->hasVscnt());
1153 BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),
1154 TII->get(AMDGPU::S_WAITCNT_VSCNT))
1155 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1156 .addImm(Wait.VsCnt);
1157 TrackedWaitcntSet.insert(SWaitInst);
1160 LLVM_DEBUG(dbgs() << "insertWaitcntInBlock\n"
1161 << "Old Instr: " << MI << '\n'
1162 << "New Instr: " << *SWaitInst << '\n');
1168 // This is a flat memory operation. Check to see if it has memory
1169 // tokens for both LDS and Memory, and if so mark it as a flat.
1170 bool SIInsertWaitcnts::mayAccessLDSThroughFlat(const MachineInstr &MI) const {
1171 if (MI.memoperands_empty())
1174 for (const MachineMemOperand *Memop : MI.memoperands()) {
1175 unsigned AS = Memop->getAddrSpace();
1176 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
1183 void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
1184 WaitcntBrackets *ScoreBrackets) {
1185 // Now look at the instruction opcode. If it is a memory access
1186 // instruction, update the upper-bound of the appropriate counter's
1187 // bracket and the destination operand scores.
1188 // TODO: Use the (TSFlags & SIInstrFlags::LGKM_CNT) property everywhere.
1189 if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
1190 if (TII->isAlwaysGDS(Inst.getOpcode()) ||
1191 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
1192 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_ACCESS, Inst);
1193 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_GPR_LOCK, Inst);
1195 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst);
1197 } else if (TII->isFLAT(Inst)) {
1198 assert(Inst.mayLoad() || Inst.mayStore());
1200 if (TII->usesVM_CNT(Inst)) {
1201 if (!ST->hasVscnt())
1202 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_ACCESS, Inst);
1203 else if (Inst.mayLoad() &&
1204 AMDGPU::getAtomicRetOp(Inst.getOpcode()) == -1)
1205 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_READ_ACCESS, Inst);
1207 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_WRITE_ACCESS, Inst);
1210 if (TII->usesLGKM_CNT(Inst)) {
1211 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst);
1213 // This is a flat memory operation, so note it - it will require
1214 // that both the VM and LGKM be flushed to zero if it is pending when
1215 // a VM or LGKM dependency occurs.
1216 if (mayAccessLDSThroughFlat(Inst))
1217 ScoreBrackets->setPendingFlat();
1219 } else if (SIInstrInfo::isVMEM(Inst) &&
1220 // TODO: get a better carve out.
1221 Inst.getOpcode() != AMDGPU::BUFFER_WBINVL1 &&
1222 Inst.getOpcode() != AMDGPU::BUFFER_WBINVL1_SC &&
1223 Inst.getOpcode() != AMDGPU::BUFFER_WBINVL1_VOL &&
1224 Inst.getOpcode() != AMDGPU::BUFFER_GL0_INV &&
1225 Inst.getOpcode() != AMDGPU::BUFFER_GL1_INV) {
1226 if (!ST->hasVscnt())
1227 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_ACCESS, Inst);
1228 else if ((Inst.mayLoad() &&
1229 AMDGPU::getAtomicRetOp(Inst.getOpcode()) == -1) ||
1230 /* IMAGE_GET_RESINFO / IMAGE_GET_LOD */
1231 (TII->isMIMG(Inst) && !Inst.mayLoad() && !Inst.mayStore()))
1232 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_READ_ACCESS, Inst);
1233 else if (Inst.mayStore())
1234 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_WRITE_ACCESS, Inst);
1236 if (ST->vmemWriteNeedsExpWaitcnt() &&
1237 (Inst.mayStore() || AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1)) {
1238 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMW_GPR_LOCK, Inst);
1240 } else if (TII->isSMRD(Inst)) {
1241 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
1242 } else if (Inst.isCall()) {
1243 if (callWaitsOnFunctionReturn(Inst)) {
1244 // Act as a wait on everything
1245 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt::allZero(IV));
1247 // May need to way wait for anything.
1248 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt());
1251 switch (Inst.getOpcode()) {
1252 case AMDGPU::S_SENDMSG:
1253 case AMDGPU::S_SENDMSGHALT:
1254 ScoreBrackets->updateByEvent(TII, TRI, MRI, SQ_MESSAGE, Inst);
1257 case AMDGPU::EXP_DONE: {
1258 int Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
1259 if (Imm >= 32 && Imm <= 63)
1260 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_PARAM_ACCESS, Inst);
1261 else if (Imm >= 12 && Imm <= 15)
1262 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_POS_ACCESS, Inst);
1264 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_GPR_LOCK, Inst);
1267 case AMDGPU::S_MEMTIME:
1268 case AMDGPU::S_MEMREALTIME:
1269 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
1277 bool WaitcntBrackets::mergeScore(const MergeInfo &M, uint32_t &Score,
1278 uint32_t OtherScore) {
1279 uint32_t MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
1280 uint32_t OtherShifted =
1281 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
1282 Score = std::max(MyShifted, OtherShifted);
1283 return OtherShifted > MyShifted;
1286 /// Merge the pending events and associater score brackets of \p Other into
1287 /// this brackets status.
1289 /// Returns whether the merge resulted in a change that requires tighter waits
1290 /// (i.e. the merged brackets strictly dominate the original brackets).
1291 bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
1292 bool StrictDom = false;
1294 for (auto T : inst_counter_types()) {
1295 // Merge event flags for this counter
1296 const bool OldOutOfOrder = counterOutOfOrder(T);
1297 const uint32_t OldEvents = PendingEvents & WaitEventMaskForInst[T];
1298 const uint32_t OtherEvents = Other.PendingEvents & WaitEventMaskForInst[T];
1299 if (OtherEvents & ~OldEvents)
1301 if (Other.MixedPendingEvents[T] ||
1302 (OldEvents && OtherEvents && OldEvents != OtherEvents))
1303 MixedPendingEvents[T] = true;
1304 PendingEvents |= OtherEvents;
1306 // Merge scores for this counter
1307 const uint32_t MyPending = ScoreUBs[T] - ScoreLBs[T];
1308 const uint32_t OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
1310 M.OldLB = ScoreLBs[T];
1311 M.OtherLB = Other.ScoreLBs[T];
1312 M.MyShift = OtherPending > MyPending ? OtherPending - MyPending : 0;
1313 M.OtherShift = ScoreUBs[T] - Other.ScoreUBs[T] + M.MyShift;
1315 const uint32_t NewUB = ScoreUBs[T] + M.MyShift;
1316 if (NewUB < ScoreUBs[T])
1317 report_fatal_error("waitcnt score overflow");
1318 ScoreUBs[T] = NewUB;
1319 ScoreLBs[T] = std::min(M.OldLB + M.MyShift, M.OtherLB + M.OtherShift);
1321 StrictDom |= mergeScore(M, LastFlat[T], Other.LastFlat[T]);
1323 bool RegStrictDom = false;
1324 for (int J = 0, E = std::max(getMaxVGPR(), Other.getMaxVGPR()) + 1; J != E;
1326 RegStrictDom |= mergeScore(M, VgprScores[T][J], Other.VgprScores[T][J]);
1329 if (T == LGKM_CNT) {
1330 for (int J = 0, E = std::max(getMaxSGPR(), Other.getMaxSGPR()) + 1;
1332 RegStrictDom |= mergeScore(M, SgprScores[J], Other.SgprScores[J]);
1336 if (RegStrictDom && !OldOutOfOrder)
1340 VgprUB = std::max(getMaxVGPR(), Other.getMaxVGPR());
1341 SgprUB = std::max(getMaxSGPR(), Other.getMaxSGPR());
1346 // Generate s_waitcnt instructions where needed.
1347 bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
1348 MachineBasicBlock &Block,
1349 WaitcntBrackets &ScoreBrackets) {
1350 bool Modified = false;
1353 dbgs() << "*** Block" << Block.getNumber() << " ***";
1354 ScoreBrackets.dump();
1357 // Walk over the instructions.
1358 MachineInstr *OldWaitcntInstr = nullptr;
1360 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
1361 E = Block.instr_end();
1363 MachineInstr &Inst = *Iter;
1365 // Track pre-existing waitcnts from earlier iterations.
1366 if (Inst.getOpcode() == AMDGPU::S_WAITCNT ||
1367 (Inst.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1368 Inst.getOperand(0).isReg() &&
1369 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL)) {
1370 if (!OldWaitcntInstr)
1371 OldWaitcntInstr = &Inst;
1376 bool VCCZBugWorkAround = false;
1377 if (readsVCCZ(Inst) &&
1378 (!VCCZBugHandledSet.count(&Inst))) {
1379 if (ScoreBrackets.getScoreLB(LGKM_CNT) <
1380 ScoreBrackets.getScoreUB(LGKM_CNT) &&
1381 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1382 if (ST->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1383 VCCZBugWorkAround = true;
1387 // Generate an s_waitcnt instruction to be placed before
1388 // cur_Inst, if needed.
1389 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr);
1390 OldWaitcntInstr = nullptr;
1392 updateEventWaitcntAfter(Inst, &ScoreBrackets);
1394 #if 0 // TODO: implement resource type check controlled by options with ub = LB.
1395 // If this instruction generates a S_SETVSKIP because it is an
1396 // indexed resource, and we are on Tahiti, then it will also force
1397 // an S_WAITCNT vmcnt(0)
1398 if (RequireCheckResourceType(Inst, context)) {
1399 // Force the score to as if an S_WAITCNT vmcnt(0) is emitted.
1400 ScoreBrackets->setScoreLB(VM_CNT,
1401 ScoreBrackets->getScoreUB(VM_CNT));
1407 ScoreBrackets.dump();
1410 // TODO: Remove this work-around after fixing the scheduler and enable the
1412 if (VCCZBugWorkAround) {
1413 // Restore the vccz bit. Any time a value is written to vcc, the vcc
1414 // bit is updated, so we can restore the bit by reading the value of
1415 // vcc and then writing it back to the register.
1416 BuildMI(Block, Inst, Inst.getDebugLoc(),
1417 TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
1419 .addReg(TRI->getVCC());
1420 VCCZBugHandledSet.insert(&Inst);
1430 bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
1431 ST = &MF.getSubtarget<GCNSubtarget>();
1432 TII = ST->getInstrInfo();
1433 TRI = &TII->getRegisterInfo();
1434 MRI = &MF.getRegInfo();
1435 IV = AMDGPU::getIsaVersion(ST->getCPU());
1436 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1438 ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
1439 for (auto T : inst_counter_types())
1440 ForceEmitWaitcnt[T] = false;
1442 HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV);
1443 HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
1444 HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);
1445 HardwareLimits.VscntMax = ST->hasVscnt() ? 63 : 0;
1447 HardwareLimits.NumVGPRsMax = ST->getAddressableNumVGPRs();
1448 HardwareLimits.NumSGPRsMax = ST->getAddressableNumSGPRs();
1449 assert(HardwareLimits.NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
1450 assert(HardwareLimits.NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
1452 RegisterEncoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0);
1453 RegisterEncoding.VGPRL =
1454 RegisterEncoding.VGPR0 + HardwareLimits.NumVGPRsMax - 1;
1455 RegisterEncoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0);
1456 RegisterEncoding.SGPRL =
1457 RegisterEncoding.SGPR0 + HardwareLimits.NumSGPRsMax - 1;
1459 TrackedWaitcntSet.clear();
1460 VCCZBugHandledSet.clear();
1464 // Keep iterating over the blocks in reverse post order, inserting and
1465 // updating s_waitcnt where needed, until a fix point is reached.
1466 for (MachineBasicBlock *MBB :
1467 ReversePostOrderTraversal<MachineFunction *>(&MF)) {
1468 RpotIdxMap[MBB] = BlockInfos.size();
1469 BlockInfos.emplace_back(MBB);
1472 std::unique_ptr<WaitcntBrackets> Brackets;
1473 bool Modified = false;
1478 for (BlockInfo &BI : BlockInfos) {
1482 unsigned Idx = std::distance(&*BlockInfos.begin(), &BI);
1486 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
1488 *Brackets = *BI.Incoming;
1491 Brackets = std::make_unique<WaitcntBrackets>(ST);
1496 Modified |= insertWaitcntInBlock(MF, *BI.MBB, *Brackets);
1499 if (Brackets->hasPending()) {
1500 BlockInfo *MoveBracketsToSucc = nullptr;
1501 for (MachineBasicBlock *Succ : BI.MBB->successors()) {
1502 unsigned SuccIdx = RpotIdxMap[Succ];
1503 BlockInfo &SuccBI = BlockInfos[SuccIdx];
1504 if (!SuccBI.Incoming) {
1505 SuccBI.Dirty = true;
1508 if (!MoveBracketsToSucc) {
1509 MoveBracketsToSucc = &SuccBI;
1511 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
1513 } else if (SuccBI.Incoming->merge(*Brackets)) {
1514 SuccBI.Dirty = true;
1519 if (MoveBracketsToSucc)
1520 MoveBracketsToSucc->Incoming = std::move(Brackets);
1525 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
1527 bool HaveScalarStores = false;
1529 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
1531 MachineBasicBlock &MBB = *BI;
1533 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
1535 if (!HaveScalarStores && TII->isScalarStore(*I))
1536 HaveScalarStores = true;
1538 if (I->getOpcode() == AMDGPU::S_ENDPGM ||
1539 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
1540 EndPgmBlocks.push_back(&MBB);
1544 if (HaveScalarStores) {
1545 // If scalar writes are used, the cache must be flushed or else the next
1546 // wave to reuse the same scratch memory can be clobbered.
1548 // Insert s_dcache_wb at wave termination points if there were any scalar
1549 // stores, and only if the cache hasn't already been flushed. This could be
1550 // improved by looking across blocks for flushes in postdominating blocks
1551 // from the stores but an explicitly requested flush is probably very rare.
1552 for (MachineBasicBlock *MBB : EndPgmBlocks) {
1553 bool SeenDCacheWB = false;
1555 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
1557 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
1558 SeenDCacheWB = true;
1559 else if (TII->isScalarStore(*I))
1560 SeenDCacheWB = false;
1562 // FIXME: It would be better to insert this before a waitcnt if any.
1563 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
1564 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
1567 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB));
1573 if (!MFI->isEntryFunction()) {
1574 // Wait for any outstanding memory operations that the input registers may
1575 // depend on. We can't track them and it's better to the wait after the
1576 // costly call sequence.
1578 // TODO: Could insert earlier and schedule more liberally with operations
1579 // that only use caller preserved registers.
1580 MachineBasicBlock &EntryBB = MF.front();
1582 BuildMI(EntryBB, EntryBB.getFirstNonPHI(), DebugLoc(),
1583 TII->get(AMDGPU::S_WAITCNT_VSCNT))
1584 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1586 BuildMI(EntryBB, EntryBB.getFirstNonPHI(), DebugLoc(), TII->get(AMDGPU::S_WAITCNT))