1 //===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10 // SGPR spills, so must insert CSR SGPR spills as well as expand them.
12 // This pass must never create new SGPR virtual registers.
14 // FIXME: Must stop RegScavenger spills in later passes.
16 //===----------------------------------------------------------------------===//
19 #include "AMDGPUSubtarget.h"
20 #include "SIInstrInfo.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "llvm/CodeGen/LiveIntervals.h"
23 #include "llvm/CodeGen/MachineBasicBlock.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstr.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineOperand.h"
29 #include "llvm/CodeGen/VirtRegMap.h"
30 #include "llvm/InitializePasses.h"
31 #include "llvm/Target/TargetMachine.h"
35 #define DEBUG_TYPE "si-lower-sgpr-spills"
37 using MBBVector = SmallVector<MachineBasicBlock *, 4>;
41 static cl::opt<bool> EnableSpillVGPRToAGPR(
42 "amdgpu-spill-vgpr-to-agpr",
43 cl::desc("Enable spilling VGPRs to AGPRs"),
47 class SILowerSGPRSpills : public MachineFunctionPass {
49 const SIRegisterInfo *TRI = nullptr;
50 const SIInstrInfo *TII = nullptr;
51 VirtRegMap *VRM = nullptr;
52 LiveIntervals *LIS = nullptr;
54 // Save and Restore blocks of the current function. Typically there is a
55 // single save block, unless Windows EH funclets are involved.
57 MBBVector RestoreBlocks;
62 SILowerSGPRSpills() : MachineFunctionPass(ID) {}
64 void calculateSaveRestoreBlocks(MachineFunction &MF);
65 bool spillCalleeSavedRegs(MachineFunction &MF);
67 bool runOnMachineFunction(MachineFunction &MF) override;
69 void getAnalysisUsage(AnalysisUsage &AU) const override {
71 MachineFunctionPass::getAnalysisUsage(AU);
75 } // end anonymous namespace
77 char SILowerSGPRSpills::ID = 0;
79 INITIALIZE_PASS_BEGIN(SILowerSGPRSpills, DEBUG_TYPE,
80 "SI lower SGPR spill instructions", false, false)
81 INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
82 INITIALIZE_PASS_END(SILowerSGPRSpills, DEBUG_TYPE,
83 "SI lower SGPR spill instructions", false, false)
85 char &llvm::SILowerSGPRSpillsID = SILowerSGPRSpills::ID;
87 /// Insert restore code for the callee-saved registers used in the function.
88 static void insertCSRSaves(MachineBasicBlock &SaveBlock,
89 ArrayRef<CalleeSavedInfo> CSI,
91 MachineFunction &MF = *SaveBlock.getParent();
92 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
93 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
94 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
96 MachineBasicBlock::iterator I = SaveBlock.begin();
97 if (!TFI->spillCalleeSavedRegisters(SaveBlock, I, CSI, TRI)) {
98 for (const CalleeSavedInfo &CS : CSI) {
99 // Insert the spill to the stack frame.
100 unsigned Reg = CS.getReg();
102 MachineInstrSpan MIS(I, &SaveBlock);
103 const TargetRegisterClass *RC =
104 TRI->getMinimalPhysRegClass(Reg, MVT::i32);
106 TII.storeRegToStackSlot(SaveBlock, I, Reg, true, CS.getFrameIdx(), RC,
110 assert(std::distance(MIS.begin(), I) == 1);
111 MachineInstr &Inst = *std::prev(I);
113 LIS->InsertMachineInstrInMaps(Inst);
114 LIS->removeAllRegUnitsForPhysReg(Reg);
120 /// Insert restore code for the callee-saved registers used in the function.
121 static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
122 MutableArrayRef<CalleeSavedInfo> CSI,
123 LiveIntervals *LIS) {
124 MachineFunction &MF = *RestoreBlock.getParent();
125 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
126 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
127 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
129 // Restore all registers immediately before the return and any
130 // terminators that precede it.
131 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
133 // FIXME: Just emit the readlane/writelane directly
134 if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) {
135 for (const CalleeSavedInfo &CI : reverse(CSI)) {
136 unsigned Reg = CI.getReg();
137 const TargetRegisterClass *RC =
138 TRI->getMinimalPhysRegClass(Reg, MVT::i32);
140 TII.loadRegFromStackSlot(RestoreBlock, I, Reg, CI.getFrameIdx(), RC, TRI);
141 assert(I != RestoreBlock.begin() &&
142 "loadRegFromStackSlot didn't insert any code!");
143 // Insert in reverse order. loadRegFromStackSlot can insert
144 // multiple instructions.
147 MachineInstr &Inst = *std::prev(I);
148 LIS->InsertMachineInstrInMaps(Inst);
149 LIS->removeAllRegUnitsForPhysReg(Reg);
155 /// Compute the sets of entry and return blocks for saving and restoring
156 /// callee-saved registers, and placing prolog and epilog code.
157 void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
158 const MachineFrameInfo &MFI = MF.getFrameInfo();
160 // Even when we do not change any CSR, we still want to insert the
161 // prologue and epilogue of the function.
162 // So set the save points for those.
164 // Use the points found by shrink-wrapping, if any.
165 if (MFI.getSavePoint()) {
166 SaveBlocks.push_back(MFI.getSavePoint());
167 assert(MFI.getRestorePoint() && "Both restore and save must be set");
168 MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
169 // If RestoreBlock does not have any successor and is not a return block
170 // then the end point is unreachable and we do not need to insert any
172 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
173 RestoreBlocks.push_back(RestoreBlock);
177 // Save refs to entry and return blocks.
178 SaveBlocks.push_back(&MF.front());
179 for (MachineBasicBlock &MBB : MF) {
180 if (MBB.isEHFuncletEntry())
181 SaveBlocks.push_back(&MBB);
182 if (MBB.isReturnBlock())
183 RestoreBlocks.push_back(&MBB);
187 bool SILowerSGPRSpills::spillCalleeSavedRegs(MachineFunction &MF) {
188 MachineRegisterInfo &MRI = MF.getRegInfo();
189 const Function &F = MF.getFunction();
190 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
191 const SIFrameLowering *TFI = ST.getFrameLowering();
192 MachineFrameInfo &MFI = MF.getFrameInfo();
193 RegScavenger *RS = nullptr;
195 // Determine which of the registers in the callee save list should be saved.
197 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
199 // Add the code to save and restore the callee saved registers.
200 if (!F.hasFnAttribute(Attribute::Naked)) {
201 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
202 // necessary for verifier liveness checks.
203 MFI.setCalleeSavedInfoValid(true);
205 std::vector<CalleeSavedInfo> CSI;
206 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
208 for (unsigned I = 0; CSRegs[I]; ++I) {
209 unsigned Reg = CSRegs[I];
210 if (SavedRegs.test(Reg)) {
211 const TargetRegisterClass *RC =
212 TRI->getMinimalPhysRegClass(Reg, MVT::i32);
213 int JunkFI = MFI.CreateStackObject(TRI->getSpillSize(*RC),
214 TRI->getSpillAlign(*RC), true);
216 CSI.push_back(CalleeSavedInfo(Reg, JunkFI));
221 for (MachineBasicBlock *SaveBlock : SaveBlocks)
222 insertCSRSaves(*SaveBlock, CSI, LIS);
224 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
225 insertCSRRestores(*RestoreBlock, CSI, LIS);
233 // Find lowest available VGPR and use it as VGPR reserved for SGPR spills.
234 static bool lowerShiftReservedVGPR(MachineFunction &MF,
235 const GCNSubtarget &ST) {
236 MachineRegisterInfo &MRI = MF.getRegInfo();
237 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
238 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
239 Register LowestAvailableVGPR, ReservedVGPR;
240 ArrayRef<MCPhysReg> AllVGPR32s = ST.getRegisterInfo()->getAllVGPR32(MF);
241 for (MCPhysReg Reg : AllVGPR32s) {
242 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) {
243 LowestAvailableVGPR = Reg;
248 if (!LowestAvailableVGPR)
251 ReservedVGPR = FuncInfo->VGPRReservedForSGPRSpill;
252 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
255 for (MachineBasicBlock &MBB : MF) {
256 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
257 if (Reg.VGPR == ReservedVGPR) {
258 MBB.removeLiveIn(ReservedVGPR);
259 MBB.addLiveIn(LowestAvailableVGPR);
261 if (FuncInfo->isCalleeSavedReg(CSRegs, LowestAvailableVGPR))
262 FI = FrameInfo.CreateSpillStackObject(4, Align(4));
264 FuncInfo->setSGPRSpillVGPRs(LowestAvailableVGPR, FI, i);
268 MBB.sortUniqueLiveIns();
274 bool SILowerSGPRSpills::runOnMachineFunction(MachineFunction &MF) {
275 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
276 TII = ST.getInstrInfo();
277 TRI = &TII->getRegisterInfo();
279 VRM = getAnalysisIfAvailable<VirtRegMap>();
281 assert(SaveBlocks.empty() && RestoreBlocks.empty());
283 // First, expose any CSR SGPR spills. This is mostly the same as what PEI
284 // does, but somewhat simpler.
285 calculateSaveRestoreBlocks(MF);
286 bool HasCSRs = spillCalleeSavedRegs(MF);
288 MachineFrameInfo &MFI = MF.getFrameInfo();
289 if (!MFI.hasStackObjects() && !HasCSRs) {
291 RestoreBlocks.clear();
295 MachineRegisterInfo &MRI = MF.getRegInfo();
296 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
297 const bool SpillVGPRToAGPR = ST.hasMAIInsts() && FuncInfo->hasSpilledVGPRs()
298 && EnableSpillVGPRToAGPR;
300 bool MadeChange = false;
302 const bool SpillToAGPR = EnableSpillVGPRToAGPR && ST.hasMAIInsts();
304 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
305 // handled as SpilledToReg in regular PrologEpilogInserter.
306 if ((TRI->spillSGPRToVGPR() && (HasCSRs || FuncInfo->hasSpilledSGPRs())) ||
308 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
309 // are spilled to VGPRs, in which case we can eliminate the stack usage.
311 // This operates under the assumption that only other SGPR spills are users
312 // of the frame index.
314 lowerShiftReservedVGPR(MF, ST);
316 for (MachineBasicBlock &MBB : MF) {
317 MachineBasicBlock::iterator Next;
318 for (auto I = MBB.begin(), E = MBB.end(); I != E; I = Next) {
319 MachineInstr &MI = *I;
322 if (SpillToAGPR && TII->isVGPRSpill(MI)) {
323 // Try to eliminate stack used by VGPR spills before frame
325 unsigned FIOp = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
326 AMDGPU::OpName::vaddr);
327 int FI = MI.getOperand(FIOp).getIndex();
329 TII->getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
330 if (FuncInfo->allocateVGPRSpillToAGPR(MF, FI,
331 TRI->isAGPR(MRI, VReg))) {
332 TRI->eliminateFrameIndex(MI, 0, FIOp, nullptr);
337 if (!TII->isSGPRSpill(MI))
340 int FI = TII->getNamedOperand(MI, AMDGPU::OpName::addr)->getIndex();
341 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
342 if (FuncInfo->allocateSGPRSpillToVGPR(MF, FI)) {
343 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(MI, FI, nullptr);
345 assert(Spilled && "failed to spill SGPR to VGPR when allocated");
350 for (MachineBasicBlock &MBB : MF) {
351 for (auto SSpill : FuncInfo->getSGPRSpillVGPRs())
352 MBB.addLiveIn(SSpill.VGPR);
354 for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
357 for (MCPhysReg Reg : FuncInfo->getAGPRSpillVGPRs())
360 MBB.sortUniqueLiveIns();
364 } else if (FuncInfo->VGPRReservedForSGPRSpill) {
365 FuncInfo->removeVGPRForSGPRSpill(FuncInfo->VGPRReservedForSGPRSpill, MF);
369 RestoreBlocks.clear();