1 //===- SIInsertHardClauses.cpp - Insert Hard Clauses ----------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// Insert s_clause instructions to form hard clauses.
12 /// Clausing load instructions can give cache coherency benefits. Before gfx10,
13 /// the hardware automatically detected "soft clauses", which were sequences of
14 /// memory instructions of the same type. In gfx10 this detection was removed,
15 /// and the s_clause instruction was introduced to explicitly mark "hard
18 /// It's the scheduler's job to form the clauses by putting similar memory
19 /// instructions next to each other. Our job is just to insert an s_clause
20 /// instruction to mark the start of each clause.
22 /// Note that hard clauses are very similar to, but logically distinct from, the
23 /// groups of instructions that have to be restartable when XNACK is enabled.
24 /// The rules are slightly different in each case. For example an s_nop
25 /// instruction breaks a restartable group, but can appear in the middle of a
26 /// hard clause. (Before gfx10 there wasn't a distinction, and both were called
27 /// "soft clauses" or just "clauses".)
29 /// The SIFormMemoryClauses pass and GCNHazardRecognizer deal with restartable
30 /// groups, not hard clauses.
32 //===----------------------------------------------------------------------===//
34 #include "AMDGPUSubtarget.h"
35 #include "SIInstrInfo.h"
36 #include "llvm/ADT/SmallVector.h"
40 #define DEBUG_TYPE "si-insert-hard-clauses"
45 // Texture, buffer, global or scratch memory instructions.
47 // Flat (not global or scratch) memory instructions.
49 // Instructions that access LDS.
51 // Scalar memory instructions.
55 LAST_REAL_HARDCLAUSE_TYPE = HARDCLAUSE_VALU,
57 // Internal instructions, which are allowed in the middle of a hard clause,
58 // except for s_waitcnt.
60 // Instructions that are not allowed in a hard clause: SALU, export, branch,
61 // message, GDS, s_waitcnt and anything else not mentioned above.
65 HardClauseType getHardClauseType(const MachineInstr &MI) {
66 // On current architectures we only get a benefit from clausing loads.
68 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI))
69 return HARDCLAUSE_VMEM;
70 if (SIInstrInfo::isFLAT(MI))
71 return HARDCLAUSE_FLAT;
73 if (SIInstrInfo::isSMRD(MI))
74 return HARDCLAUSE_SMEM;
77 // Don't form VALU clauses. It's not clear what benefit they give, if any.
79 // In practice s_nop is the only internal instruction we're likely to see.
80 // It's safe to treat the rest as illegal.
81 if (MI.getOpcode() == AMDGPU::S_NOP)
82 return HARDCLAUSE_INTERNAL;
83 return HARDCLAUSE_ILLEGAL;
86 class SIInsertHardClauses : public MachineFunctionPass {
90 SIInsertHardClauses() : MachineFunctionPass(ID) {}
92 void getAnalysisUsage(AnalysisUsage &AU) const override {
94 MachineFunctionPass::getAnalysisUsage(AU);
97 // Track information about a clause as we discover it.
99 // The type of all (non-internal) instructions in the clause.
100 HardClauseType Type = HARDCLAUSE_ILLEGAL;
101 // The first (necessarily non-internal) instruction in the clause.
102 MachineInstr *First = nullptr;
103 // The last non-internal instruction in the clause.
104 MachineInstr *Last = nullptr;
105 // The length of the clause including any internal instructions in the
106 // middle or after the end of the clause.
108 // The base operands of *Last.
109 SmallVector<const MachineOperand *, 4> BaseOps;
112 bool emitClause(const ClauseInfo &CI, const SIInstrInfo *SII) {
113 // Get the size of the clause excluding any internal instructions at the
116 std::distance(CI.First->getIterator(), CI.Last->getIterator()) + 1;
119 assert(Size <= 64 && "Hard clause is too long!");
121 auto &MBB = *CI.First->getParent();
123 BuildMI(MBB, *CI.First, DebugLoc(), SII->get(AMDGPU::S_CLAUSE))
125 finalizeBundle(MBB, ClauseMI->getIterator(),
126 std::next(CI.Last->getIterator()));
130 bool runOnMachineFunction(MachineFunction &MF) override {
131 if (skipFunction(MF.getFunction()))
134 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
135 if (!ST.hasHardClauses())
138 const SIInstrInfo *SII = ST.getInstrInfo();
139 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
141 bool Changed = false;
142 for (auto &MBB : MF) {
144 for (auto &MI : MBB) {
145 HardClauseType Type = getHardClauseType(MI);
150 SmallVector<const MachineOperand *, 4> BaseOps;
151 if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
152 if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2,
154 // We failed to get the base operands, so we'll never clause this
155 // instruction with any other, so pretend it's illegal.
156 Type = HARDCLAUSE_ILLEGAL;
160 if (CI.Length == 64 ||
161 (CI.Length && Type != HARDCLAUSE_INTERNAL &&
163 // Note that we lie to shouldClusterMemOps about the size of the
164 // cluster. When shouldClusterMemOps is called from the machine
165 // scheduler it limits the size of the cluster to avoid increasing
166 // register pressure too much, but this pass runs after register
167 // allocation so there is no need for that kind of limit.
168 !SII->shouldClusterMemOps(CI.BaseOps, BaseOps, 2, 2)))) {
169 // Finish the current clause.
170 Changed |= emitClause(CI, SII);
175 // Extend the current clause.
177 if (Type != HARDCLAUSE_INTERNAL) {
179 CI.BaseOps = std::move(BaseOps);
181 } else if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
182 // Start a new clause.
183 CI = ClauseInfo{Type, &MI, &MI, 1, std::move(BaseOps)};
187 // Finish the last clause in the basic block if any.
189 Changed |= emitClause(CI, SII);
198 char SIInsertHardClauses::ID = 0;
200 char &llvm::SIInsertHardClausesID = SIInsertHardClauses::ID;
202 INITIALIZE_PASS(SIInsertHardClauses, DEBUG_TYPE, "SI Insert Hard Clauses",