1 //===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 #include "AMDGPUSubtarget.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "SIInstrInfo.h"
13 #include "llvm/ADT/SmallSet.h"
14 #include "llvm/CodeGen/MachineFunctionPass.h"
15 #include "llvm/CodeGen/MachineInstrBuilder.h"
16 #include "llvm/CodeGen/MachineRegisterInfo.h"
17 #include "llvm/InitializePasses.h"
18 #include "llvm/Support/Debug.h"
22 #define DEBUG_TYPE "si-optimize-exec-masking"
26 class SIOptimizeExecMasking : public MachineFunctionPass {
31 SIOptimizeExecMasking() : MachineFunctionPass(ID) {
32 initializeSIOptimizeExecMaskingPass(*PassRegistry::getPassRegistry());
35 bool runOnMachineFunction(MachineFunction &MF) override;
37 StringRef getPassName() const override {
38 return "SI optimize exec mask operations";
41 void getAnalysisUsage(AnalysisUsage &AU) const override {
43 MachineFunctionPass::getAnalysisUsage(AU);
47 } // End anonymous namespace.
49 INITIALIZE_PASS_BEGIN(SIOptimizeExecMasking, DEBUG_TYPE,
50 "SI optimize exec mask operations", false, false)
51 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
52 INITIALIZE_PASS_END(SIOptimizeExecMasking, DEBUG_TYPE,
53 "SI optimize exec mask operations", false, false)
55 char SIOptimizeExecMasking::ID = 0;
57 char &llvm::SIOptimizeExecMaskingID = SIOptimizeExecMasking::ID;
59 /// If \p MI is a copy from exec, return the register copied to.
60 static Register isCopyFromExec(const MachineInstr &MI, const GCNSubtarget &ST) {
61 switch (MI.getOpcode()) {
63 case AMDGPU::S_MOV_B64:
64 case AMDGPU::S_MOV_B64_term:
65 case AMDGPU::S_MOV_B32:
66 case AMDGPU::S_MOV_B32_term: {
67 const MachineOperand &Src = MI.getOperand(1);
69 Src.getReg() == (ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC))
70 return MI.getOperand(0).getReg();
74 return AMDGPU::NoRegister;
77 /// If \p MI is a copy to exec, return the register copied from.
78 static Register isCopyToExec(const MachineInstr &MI, const GCNSubtarget &ST) {
79 switch (MI.getOpcode()) {
81 case AMDGPU::S_MOV_B64:
82 case AMDGPU::S_MOV_B32: {
83 const MachineOperand &Dst = MI.getOperand(0);
85 Dst.getReg() == (ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC) &&
86 MI.getOperand(1).isReg())
87 return MI.getOperand(1).getReg();
90 case AMDGPU::S_MOV_B64_term:
91 case AMDGPU::S_MOV_B32_term:
92 llvm_unreachable("should have been replaced");
98 /// If \p MI is a logical operation on an exec value,
99 /// return the register copied to.
100 static Register isLogicalOpOnExec(const MachineInstr &MI) {
101 switch (MI.getOpcode()) {
102 case AMDGPU::S_AND_B64:
103 case AMDGPU::S_OR_B64:
104 case AMDGPU::S_XOR_B64:
105 case AMDGPU::S_ANDN2_B64:
106 case AMDGPU::S_ORN2_B64:
107 case AMDGPU::S_NAND_B64:
108 case AMDGPU::S_NOR_B64:
109 case AMDGPU::S_XNOR_B64: {
110 const MachineOperand &Src1 = MI.getOperand(1);
111 if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC)
112 return MI.getOperand(0).getReg();
113 const MachineOperand &Src2 = MI.getOperand(2);
114 if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC)
115 return MI.getOperand(0).getReg();
118 case AMDGPU::S_AND_B32:
119 case AMDGPU::S_OR_B32:
120 case AMDGPU::S_XOR_B32:
121 case AMDGPU::S_ANDN2_B32:
122 case AMDGPU::S_ORN2_B32:
123 case AMDGPU::S_NAND_B32:
124 case AMDGPU::S_NOR_B32:
125 case AMDGPU::S_XNOR_B32: {
126 const MachineOperand &Src1 = MI.getOperand(1);
127 if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC_LO)
128 return MI.getOperand(0).getReg();
129 const MachineOperand &Src2 = MI.getOperand(2);
130 if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC_LO)
131 return MI.getOperand(0).getReg();
136 return AMDGPU::NoRegister;
139 static unsigned getSaveExecOp(unsigned Opc) {
141 case AMDGPU::S_AND_B64:
142 return AMDGPU::S_AND_SAVEEXEC_B64;
143 case AMDGPU::S_OR_B64:
144 return AMDGPU::S_OR_SAVEEXEC_B64;
145 case AMDGPU::S_XOR_B64:
146 return AMDGPU::S_XOR_SAVEEXEC_B64;
147 case AMDGPU::S_ANDN2_B64:
148 return AMDGPU::S_ANDN2_SAVEEXEC_B64;
149 case AMDGPU::S_ORN2_B64:
150 return AMDGPU::S_ORN2_SAVEEXEC_B64;
151 case AMDGPU::S_NAND_B64:
152 return AMDGPU::S_NAND_SAVEEXEC_B64;
153 case AMDGPU::S_NOR_B64:
154 return AMDGPU::S_NOR_SAVEEXEC_B64;
155 case AMDGPU::S_XNOR_B64:
156 return AMDGPU::S_XNOR_SAVEEXEC_B64;
157 case AMDGPU::S_AND_B32:
158 return AMDGPU::S_AND_SAVEEXEC_B32;
159 case AMDGPU::S_OR_B32:
160 return AMDGPU::S_OR_SAVEEXEC_B32;
161 case AMDGPU::S_XOR_B32:
162 return AMDGPU::S_XOR_SAVEEXEC_B32;
163 case AMDGPU::S_ANDN2_B32:
164 return AMDGPU::S_ANDN2_SAVEEXEC_B32;
165 case AMDGPU::S_ORN2_B32:
166 return AMDGPU::S_ORN2_SAVEEXEC_B32;
167 case AMDGPU::S_NAND_B32:
168 return AMDGPU::S_NAND_SAVEEXEC_B32;
169 case AMDGPU::S_NOR_B32:
170 return AMDGPU::S_NOR_SAVEEXEC_B32;
171 case AMDGPU::S_XNOR_B32:
172 return AMDGPU::S_XNOR_SAVEEXEC_B32;
174 return AMDGPU::INSTRUCTION_LIST_END;
178 // These are only terminators to get correct spill code placement during
179 // register allocation, so turn them back into normal instructions. Only one of
180 // these is expected per block.
181 static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
182 switch (MI.getOpcode()) {
183 case AMDGPU::S_MOV_B64_term:
184 case AMDGPU::S_MOV_B32_term: {
185 MI.setDesc(TII.get(AMDGPU::COPY));
188 case AMDGPU::S_XOR_B64_term: {
189 // This is only a terminator to get the correct spill code placement during
190 // register allocation.
191 MI.setDesc(TII.get(AMDGPU::S_XOR_B64));
194 case AMDGPU::S_XOR_B32_term: {
195 // This is only a terminator to get the correct spill code placement during
196 // register allocation.
197 MI.setDesc(TII.get(AMDGPU::S_XOR_B32));
200 case AMDGPU::S_OR_B32_term: {
201 // This is only a terminator to get the correct spill code placement during
202 // register allocation.
203 MI.setDesc(TII.get(AMDGPU::S_OR_B32));
206 case AMDGPU::S_ANDN2_B64_term: {
207 // This is only a terminator to get the correct spill code placement during
208 // register allocation.
209 MI.setDesc(TII.get(AMDGPU::S_ANDN2_B64));
212 case AMDGPU::S_ANDN2_B32_term: {
213 // This is only a terminator to get the correct spill code placement during
214 // register allocation.
215 MI.setDesc(TII.get(AMDGPU::S_ANDN2_B32));
223 static MachineBasicBlock::reverse_iterator fixTerminators(
224 const SIInstrInfo &TII,
225 MachineBasicBlock &MBB) {
226 MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
227 for (; I != E; ++I) {
228 if (!I->isTerminator())
231 if (removeTerminatorBit(TII, *I))
238 static MachineBasicBlock::reverse_iterator findExecCopy(
239 const SIInstrInfo &TII,
240 const GCNSubtarget &ST,
241 MachineBasicBlock &MBB,
242 MachineBasicBlock::reverse_iterator I,
243 unsigned CopyToExec) {
244 const unsigned InstLimit = 25;
247 for (unsigned N = 0; N <= InstLimit && I != E; ++I, ++N) {
248 Register CopyFromExec = isCopyFromExec(*I, ST);
249 if (CopyFromExec.isValid())
256 // XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
257 // report the register as unavailable because a super-register with a lane mask
259 static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
260 for (MachineBasicBlock *Succ : MBB.successors()) {
261 if (Succ->isLiveIn(Reg))
268 bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
269 if (skipFunction(MF.getFunction()))
272 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
273 const SIRegisterInfo *TRI = ST.getRegisterInfo();
274 const SIInstrInfo *TII = ST.getInstrInfo();
275 MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
277 // Optimize sequences emitted for control flow lowering. They are originally
278 // emitted as the separate operations because spill code may need to be
279 // inserted for the saved copy of exec.
282 // z = s_<op>_b64 x, y
285 // x = s_<op>_saveexec_b64 y
288 for (MachineBasicBlock &MBB : MF) {
289 MachineBasicBlock::reverse_iterator I = fixTerminators(*TII, MBB);
290 MachineBasicBlock::reverse_iterator E = MBB.rend();
294 Register CopyToExec = isCopyToExec(*I, ST);
295 if (!CopyToExec.isValid())
298 // Scan backwards to find the def.
299 auto CopyToExecInst = &*I;
300 auto CopyFromExecInst = findExecCopy(*TII, ST, MBB, I, CopyToExec);
301 if (CopyFromExecInst == E) {
302 auto PrepareExecInst = std::next(I);
303 if (PrepareExecInst == E)
305 // Fold exec = COPY (S_AND_B64 reg, exec) -> exec = S_AND_B64 reg, exec
306 if (CopyToExecInst->getOperand(1).isKill() &&
307 isLogicalOpOnExec(*PrepareExecInst) == CopyToExec) {
308 LLVM_DEBUG(dbgs() << "Fold exec copy: " << *PrepareExecInst);
310 PrepareExecInst->getOperand(0).setReg(Exec);
312 LLVM_DEBUG(dbgs() << "into: " << *PrepareExecInst << '\n');
314 CopyToExecInst->eraseFromParent();
320 if (isLiveOut(MBB, CopyToExec)) {
321 // The copied register is live out and has a second use in another block.
322 LLVM_DEBUG(dbgs() << "Exec copy source register is live out\n");
326 Register CopyFromExec = CopyFromExecInst->getOperand(0).getReg();
327 MachineInstr *SaveExecInst = nullptr;
328 SmallVector<MachineInstr *, 4> OtherUseInsts;
330 for (MachineBasicBlock::iterator J
331 = std::next(CopyFromExecInst->getIterator()), JE = I->getIterator();
333 if (SaveExecInst && J->readsRegister(Exec, TRI)) {
334 LLVM_DEBUG(dbgs() << "exec read prevents saveexec: " << *J << '\n');
335 // Make sure this is inserted after any VALU ops that may have been
336 // scheduled in between.
337 SaveExecInst = nullptr;
341 bool ReadsCopyFromExec = J->readsRegister(CopyFromExec, TRI);
343 if (J->modifiesRegister(CopyToExec, TRI)) {
345 LLVM_DEBUG(dbgs() << "Multiple instructions modify "
346 << printReg(CopyToExec, TRI) << '\n');
347 SaveExecInst = nullptr;
351 unsigned SaveExecOp = getSaveExecOp(J->getOpcode());
352 if (SaveExecOp == AMDGPU::INSTRUCTION_LIST_END)
355 if (ReadsCopyFromExec) {
357 LLVM_DEBUG(dbgs() << "Found save exec op: " << *SaveExecInst << '\n');
361 << "Instruction does not read exec copy: " << *J << '\n');
364 } else if (ReadsCopyFromExec && !SaveExecInst) {
365 // Make sure no other instruction is trying to use this copy, before it
366 // will be rewritten by the saveexec, i.e. hasOneUse. There may have
367 // been another use, such as an inserted spill. For example:
369 // %sgpr0_sgpr1 = COPY %exec
370 // spill %sgpr0_sgpr1
371 // %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1
373 LLVM_DEBUG(dbgs() << "Found second use of save inst candidate: " << *J
378 if (SaveExecInst && J->readsRegister(CopyToExec, TRI)) {
379 assert(SaveExecInst != &*J);
380 OtherUseInsts.push_back(&*J);
387 LLVM_DEBUG(dbgs() << "Insert save exec op: " << *SaveExecInst << '\n');
389 MachineOperand &Src0 = SaveExecInst->getOperand(1);
390 MachineOperand &Src1 = SaveExecInst->getOperand(2);
392 MachineOperand *OtherOp = nullptr;
394 if (Src0.isReg() && Src0.getReg() == CopyFromExec) {
396 } else if (Src1.isReg() && Src1.getReg() == CopyFromExec) {
397 if (!SaveExecInst->isCommutable())
402 llvm_unreachable("unexpected");
404 CopyFromExecInst->eraseFromParent();
406 auto InsPt = SaveExecInst->getIterator();
407 const DebugLoc &DL = SaveExecInst->getDebugLoc();
409 BuildMI(MBB, InsPt, DL, TII->get(getSaveExecOp(SaveExecInst->getOpcode())),
411 .addReg(OtherOp->getReg());
412 SaveExecInst->eraseFromParent();
414 CopyToExecInst->eraseFromParent();
416 for (MachineInstr *OtherInst : OtherUseInsts) {
417 OtherInst->substituteRegister(CopyToExec, Exec,
418 AMDGPU::NoSubRegister, *TRI);