1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUArgumentUsageInfo.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIRegisterInfo.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/ADT/Optional.h"
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/IR/CallingConv.h"
21 #include "llvm/IR/Function.h"
29 SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
30 : AMDGPUMachineFunction(MF),
31 PrivateSegmentBuffer(false),
34 KernargSegmentPtr(false),
36 FlatScratchInit(false),
41 PrivateSegmentWaveByteOffset(false),
45 ImplicitBufferPtr(false),
46 ImplicitArgPtr(false),
47 GITPtrHigh(0xffffffff),
48 HighBitsOf32BitAddress(0),
50 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
51 const Function &F = MF.getFunction();
52 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
53 WavesPerEU = ST.getWavesPerEU(F);
55 Occupancy = ST.computeOccupancy(MF, getLDSSize());
56 CallingConv::ID CC = F.getCallingConv();
58 if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL) {
60 KernargSegmentPtr = true;
63 } else if (CC == CallingConv::AMDGPU_PS) {
64 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
67 if (!isEntryFunction()) {
68 // Non-entry functions have no special inputs for now, other registers
69 // required for scratch access.
70 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
71 ScratchWaveOffsetReg = AMDGPU::SGPR33;
73 // TODO: Pick a high register, and shift down, similar to a kernel.
74 FrameOffsetReg = AMDGPU::SGPR34;
75 StackPtrOffsetReg = AMDGPU::SGPR32;
77 ArgInfo.PrivateSegmentBuffer =
78 ArgDescriptor::createRegister(ScratchRSrcReg);
79 ArgInfo.PrivateSegmentWaveByteOffset =
80 ArgDescriptor::createRegister(ScratchWaveOffsetReg);
82 if (F.hasFnAttribute("amdgpu-implicitarg-ptr"))
83 ImplicitArgPtr = true;
85 if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) {
86 KernargSegmentPtr = true;
87 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
92 if (F.hasFnAttribute("amdgpu-work-group-id-x"))
95 if (F.hasFnAttribute("amdgpu-work-group-id-y"))
98 if (F.hasFnAttribute("amdgpu-work-group-id-z"))
101 if (F.hasFnAttribute("amdgpu-work-item-id-x"))
104 if (F.hasFnAttribute("amdgpu-work-item-id-y"))
107 if (F.hasFnAttribute("amdgpu-work-item-id-z"))
110 const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
111 bool HasStackObjects = FrameInfo.hasStackObjects();
113 if (isEntryFunction()) {
114 // X, XY, and XYZ are the only supported combinations, so make sure Y is
119 PrivateSegmentWaveByteOffset = true;
121 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
122 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
123 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
124 ArgInfo.PrivateSegmentWaveByteOffset =
125 ArgDescriptor::createRegister(AMDGPU::SGPR5);
128 bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
129 if (isAmdHsaOrMesa) {
130 PrivateSegmentBuffer = true;
132 if (F.hasFnAttribute("amdgpu-dispatch-ptr"))
135 if (F.hasFnAttribute("amdgpu-queue-ptr"))
138 if (F.hasFnAttribute("amdgpu-dispatch-id"))
140 } else if (ST.isMesaGfxShader(F)) {
141 ImplicitBufferPtr = true;
144 if (F.hasFnAttribute("amdgpu-kernarg-segment-ptr"))
145 KernargSegmentPtr = true;
147 if (ST.hasFlatAddressSpace() && isEntryFunction() && isAmdHsaOrMesa) {
148 auto hasNonSpillStackObjects = [&]() {
149 // Avoid expensive checking if there's no stack objects.
150 if (!HasStackObjects)
152 for (auto OI = FrameInfo.getObjectIndexBegin(),
153 OE = FrameInfo.getObjectIndexEnd(); OI != OE; ++OI)
154 if (!FrameInfo.isSpillSlotObjectIndex(OI))
156 // All stack objects are spill slots.
159 // TODO: This could be refined a lot. The attribute is a poor way of
160 // detecting calls that may require it before argument lowering.
161 if (hasNonSpillStackObjects() || F.hasFnAttribute("amdgpu-flat-scratch"))
162 FlatScratchInit = true;
165 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
166 StringRef S = A.getValueAsString();
168 S.consumeInteger(0, GITPtrHigh);
170 A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
171 S = A.getValueAsString();
173 S.consumeInteger(0, HighBitsOf32BitAddress);
175 S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();
177 S.consumeInteger(0, GDSSize);
180 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
181 limitOccupancy(getMaxWavesPerEU());
182 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
183 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
187 unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer(
188 const SIRegisterInfo &TRI) {
189 ArgInfo.PrivateSegmentBuffer =
190 ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
191 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
193 return ArgInfo.PrivateSegmentBuffer.getRegister();
196 unsigned SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
197 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
198 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
200 return ArgInfo.DispatchPtr.getRegister();
203 unsigned SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
204 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
205 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
207 return ArgInfo.QueuePtr.getRegister();
210 unsigned SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
211 ArgInfo.KernargSegmentPtr
212 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
213 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
215 return ArgInfo.KernargSegmentPtr.getRegister();
218 unsigned SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
219 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
220 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
222 return ArgInfo.DispatchID.getRegister();
225 unsigned SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
226 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
227 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
229 return ArgInfo.FlatScratchInit.getRegister();
232 unsigned SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
233 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
234 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
236 return ArgInfo.ImplicitBufferPtr.getRegister();
239 static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) {
240 for (unsigned I = 0; CSRegs[I]; ++I) {
241 if (CSRegs[I] == Reg)
248 /// \p returns true if \p NumLanes slots are available in VGPRs already used for
251 // FIXME: This only works after processFunctionBeforeFrameFinalized
252 bool SIMachineFunctionInfo::haveFreeLanesForSGPRSpill(const MachineFunction &MF,
253 unsigned NumNeed) const {
254 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
255 unsigned WaveSize = ST.getWavefrontSize();
256 return NumVGPRSpillLanes + NumNeed <= WaveSize * SpillVGPRs.size();
259 /// Reserve a slice of a VGPR to support spilling for FrameIndex \p FI.
260 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
262 std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI];
264 // This has already been allocated.
265 if (!SpillLanes.empty())
268 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
269 const SIRegisterInfo *TRI = ST.getRegisterInfo();
270 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
271 MachineRegisterInfo &MRI = MF.getRegInfo();
272 unsigned WaveSize = ST.getWavefrontSize();
274 unsigned Size = FrameInfo.getObjectSize(FI);
275 assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
276 assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs");
278 int NumLanes = Size / 4;
280 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
282 // Make sure to handle the case where a wide SGPR spill may span between two
284 for (int I = 0; I < NumLanes; ++I, ++NumVGPRSpillLanes) {
286 unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
288 if (VGPRIndex == 0) {
289 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
290 if (LaneVGPR == AMDGPU::NoRegister) {
291 // We have no VGPRs left for spilling SGPRs. Reset because we will not
292 // partially spill the SGPR to VGPRs.
293 SGPRToVGPRSpills.erase(FI);
294 NumVGPRSpillLanes -= I;
298 Optional<int> CSRSpillFI;
299 if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs &&
300 isCalleeSavedReg(CSRegs, LaneVGPR)) {
301 CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4);
304 SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI));
306 // Add this register as live-in to all blocks to avoid machine verifer
307 // complaining about use of an undefined physical register.
308 for (MachineBasicBlock &BB : MF)
309 BB.addLiveIn(LaneVGPR);
311 LaneVGPR = SpillVGPRs.back().VGPR;
314 SpillLanes.push_back(SpilledReg(LaneVGPR, VGPRIndex));
320 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
321 /// Either AGPR is spilled to VGPR to vice versa.
322 /// Returns true if a \p FI can be eliminated completely.
323 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
326 MachineRegisterInfo &MRI = MF.getRegInfo();
327 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
328 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
330 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
332 auto &Spill = VGPRToAGPRSpills[FI];
334 // This has already been allocated.
335 if (!Spill.Lanes.empty())
336 return Spill.FullyAllocated;
338 unsigned Size = FrameInfo.getObjectSize(FI);
339 unsigned NumLanes = Size / 4;
340 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
342 const TargetRegisterClass &RC =
343 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
344 auto Regs = RC.getRegisters();
346 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
347 const SIRegisterInfo *TRI = ST.getRegisterInfo();
348 Spill.FullyAllocated = true;
350 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
352 BitVector OtherUsedRegs;
353 OtherUsedRegs.resize(TRI->getNumRegs());
355 const uint32_t *CSRMask =
356 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
358 OtherUsedRegs.setBitsInMask(CSRMask);
360 // TODO: Should include register tuples, but doesn't matter with current
362 for (MCPhysReg Reg : SpillAGPR)
363 OtherUsedRegs.set(Reg);
364 for (MCPhysReg Reg : SpillVGPR)
365 OtherUsedRegs.set(Reg);
367 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
368 for (unsigned I = 0; I < NumLanes; ++I) {
369 NextSpillReg = std::find_if(
370 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
371 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
375 if (NextSpillReg == Regs.end()) { // Registers exhausted
376 Spill.FullyAllocated = false;
380 OtherUsedRegs.set(*NextSpillReg);
381 SpillRegs.push_back(*NextSpillReg);
382 Spill.Lanes[I] = *NextSpillReg++;
385 return Spill.FullyAllocated;
388 void SIMachineFunctionInfo::removeDeadFrameIndices(MachineFrameInfo &MFI) {
389 // The FP spill hasn't been inserted yet, so keep it around.
390 for (auto &R : SGPRToVGPRSpills) {
391 if (R.first != FramePointerSaveIndex)
392 MFI.RemoveStackObject(R.first);
395 // All other SPGRs must be allocated on the default stack, so reset the stack
397 for (int i = MFI.getObjectIndexBegin(), e = MFI.getObjectIndexEnd(); i != e;
399 if (i != FramePointerSaveIndex)
400 MFI.setStackID(i, TargetStackID::Default);
402 for (auto &R : VGPRToAGPRSpills) {
403 if (R.second.FullyAllocated)
404 MFI.RemoveStackObject(R.first);
408 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
409 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
410 return AMDGPU::SGPR0 + NumUserSGPRs;
413 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
414 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
417 static yaml::StringValue regToString(unsigned Reg,
418 const TargetRegisterInfo &TRI) {
419 yaml::StringValue Dest;
421 raw_string_ostream OS(Dest.Value);
422 OS << printReg(Reg, &TRI);
427 static Optional<yaml::SIArgumentInfo>
428 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
429 const TargetRegisterInfo &TRI) {
430 yaml::SIArgumentInfo AI;
432 auto convertArg = [&](Optional<yaml::SIArgument> &A,
433 const ArgDescriptor &Arg) {
437 // Create a register or stack argument.
438 yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
439 if (Arg.isRegister()) {
440 raw_string_ostream OS(SA.RegisterName.Value);
441 OS << printReg(Arg.getRegister(), &TRI);
443 SA.StackOffset = Arg.getStackOffset();
444 // Check and update the optional mask.
446 SA.Mask = Arg.getMask();
453 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
454 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
455 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
456 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
457 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
458 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
459 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
460 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
461 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
462 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
463 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
464 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
465 ArgInfo.PrivateSegmentWaveByteOffset);
466 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
467 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
468 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
469 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
470 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
478 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
479 const llvm::SIMachineFunctionInfo& MFI,
480 const TargetRegisterInfo &TRI)
481 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
482 MaxKernArgAlign(MFI.getMaxKernArgAlign()),
483 LDSSize(MFI.getLDSSize()),
484 IsEntryFunction(MFI.isEntryFunction()),
485 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
486 MemoryBound(MFI.isMemoryBound()),
487 WaveLimiter(MFI.needsWaveLimiter()),
488 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
489 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
490 ScratchWaveOffsetReg(regToString(MFI.getScratchWaveOffsetReg(), TRI)),
491 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
492 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
493 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
494 Mode(MFI.getMode()) {}
496 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
497 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
500 bool SIMachineFunctionInfo::initializeBaseYamlFields(
501 const yaml::SIMachineFunctionInfo &YamlMFI) {
502 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
503 MaxKernArgAlign = assumeAligned(YamlMFI.MaxKernArgAlign);
504 LDSSize = YamlMFI.LDSSize;
505 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
506 IsEntryFunction = YamlMFI.IsEntryFunction;
507 NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
508 MemoryBound = YamlMFI.MemoryBound;
509 WaveLimiter = YamlMFI.WaveLimiter;