1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the AArch64 implementation of TargetFrameLowering class.
12 // On AArch64, stack frames are structured as follows:
14 // The stack grows downward.
16 // All of the individual frame areas on the frame below are optional, i.e. it's
17 // possible to create a function so that the particular area isn't present
20 // At function entry, the "frame" looks as follows:
23 // |-----------------------------------|
25 // | arguments passed on the stack |
27 // |-----------------------------------| <- sp
31 // After the prologue has run, the frame has the following general structure.
32 // Note that this doesn't depict the case where a red-zone is used. Also,
33 // technically the last frame area (VLAs) doesn't get created until in the
34 // main function body, after the prologue is run. However, it's depicted here
38 // |-----------------------------------|
40 // | arguments passed on the stack |
42 // |-----------------------------------|
44 // | prev_fp, prev_lr |
45 // | (a.k.a. "frame record") |
46 // |-----------------------------------| <- fp(=x29)
48 // | other callee-saved registers |
50 // |-----------------------------------|
51 // |.empty.space.to.make.part.below....|
52 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
53 // |.the.standard.16-byte.alignment....| compile time; if present)
54 // |-----------------------------------|
56 // | local variables of fixed size |
57 // | including spill slots |
58 // |-----------------------------------| <- bp(not defined by ABI,
59 // |.variable-sized.local.variables....| LLVM chooses X19)
60 // |.(VLAs)............................| (size of this area is unknown at
61 // |...................................| compile time)
62 // |-----------------------------------| <- sp
66 // To access the data in a frame, at-compile time, a constant offset must be
67 // computable from one of the pointers (fp, bp, sp) to access it. The size
68 // of the areas with a dotted background cannot be computed at compile-time
69 // if they are present, making it required to have all three of fp, bp and
70 // sp to be set up to be able to access all contents in the frame areas,
71 // assuming all of the frame areas are non-empty.
73 // For most functions, some of the frame areas are empty. For those functions,
74 // it may not be necessary to set up fp or bp:
75 // * A base pointer is definitely needed when there are both VLAs and local
76 // variables with more-than-default alignment requirements.
77 // * A frame pointer is definitely needed when there are local variables with
78 // more-than-default alignment requirements.
80 // In some cases when a base pointer is not strictly needed, it is generated
81 // anyway when offsets from the frame pointer to access local variables become
82 // so large that the offset can't be encoded in the immediate fields of loads
85 // FIXME: also explain the redzone concept.
86 // FIXME: also explain the concept of reserved call frames.
88 //===----------------------------------------------------------------------===//
90 #include "AArch64FrameLowering.h"
91 #include "AArch64InstrInfo.h"
92 #include "AArch64MachineFunctionInfo.h"
93 #include "AArch64Subtarget.h"
94 #include "AArch64TargetMachine.h"
95 #include "llvm/ADT/Statistic.h"
96 #include "llvm/CodeGen/LivePhysRegs.h"
97 #include "llvm/CodeGen/MachineFrameInfo.h"
98 #include "llvm/CodeGen/MachineFunction.h"
99 #include "llvm/CodeGen/MachineInstrBuilder.h"
100 #include "llvm/CodeGen/MachineModuleInfo.h"
101 #include "llvm/CodeGen/MachineRegisterInfo.h"
102 #include "llvm/CodeGen/RegisterScavenging.h"
103 #include "llvm/IR/DataLayout.h"
104 #include "llvm/IR/Function.h"
105 #include "llvm/Support/CommandLine.h"
106 #include "llvm/Support/Debug.h"
107 #include "llvm/Support/raw_ostream.h"
109 using namespace llvm;
111 #define DEBUG_TYPE "frame-info"
113 static cl::opt<bool> EnableRedZone("aarch64-redzone",
114 cl::desc("enable use of redzone on AArch64"),
115 cl::init(false), cl::Hidden);
117 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
119 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
122 // Don't use the red zone if the function explicitly asks us not to.
123 // This is typically used for kernel code.
124 if (MF.getFunction()->hasFnAttribute(Attribute::NoRedZone))
127 const MachineFrameInfo &MFI = MF.getFrameInfo();
128 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
129 unsigned NumBytes = AFI->getLocalStackSize();
131 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
134 /// hasFP - Return true if the specified function should have a dedicated frame
135 /// pointer register.
136 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
137 const MachineFrameInfo &MFI = MF.getFrameInfo();
138 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
139 // Retain behavior of always omitting the FP for leaf functions when possible.
140 return (MFI.hasCalls() &&
141 MF.getTarget().Options.DisableFramePointerElim(MF)) ||
142 MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
143 MFI.hasStackMap() || MFI.hasPatchPoint() ||
144 RegInfo->needsStackRealignment(MF);
147 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
148 /// not required, we reserve argument space for call sites in the function
149 /// immediately on entry to the current function. This eliminates the need for
150 /// add/sub sp brackets around call sites. Returns true if the call frame is
151 /// included as part of the stack frame.
153 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
154 return !MF.getFrameInfo().hasVarSizedObjects();
157 MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
158 MachineFunction &MF, MachineBasicBlock &MBB,
159 MachineBasicBlock::iterator I) const {
160 const AArch64InstrInfo *TII =
161 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
162 DebugLoc DL = I->getDebugLoc();
163 unsigned Opc = I->getOpcode();
164 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
165 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
167 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
168 if (!TFI->hasReservedCallFrame(MF)) {
169 unsigned Align = getStackAlignment();
171 int64_t Amount = I->getOperand(0).getImm();
172 Amount = alignTo(Amount, Align);
176 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
177 // doesn't have to pop anything), then the first operand will be zero too so
178 // this adjustment is a no-op.
179 if (CalleePopAmount == 0) {
180 // FIXME: in-function stack adjustment for calls is limited to 24-bits
181 // because there's no guaranteed temporary register available.
183 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
184 // 1) For offset <= 12-bit, we use LSL #0
185 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
186 // LSL #0, and the other uses LSL #12.
188 // Most call frames will be allocated at the start of a function so
189 // this is OK, but it is a limitation that needs dealing with.
190 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
191 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
193 } else if (CalleePopAmount != 0) {
194 // If the calling convention demands that the callee pops arguments from the
195 // stack, we want to add it back if we have a reserved call frame.
196 assert(CalleePopAmount < 0xffffff && "call frame too large");
197 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
203 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
204 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
205 MachineFunction &MF = *MBB.getParent();
206 MachineFrameInfo &MFI = MF.getFrameInfo();
207 const TargetSubtargetInfo &STI = MF.getSubtarget();
208 const MCRegisterInfo *MRI = STI.getRegisterInfo();
209 const TargetInstrInfo *TII = STI.getInstrInfo();
210 DebugLoc DL = MBB.findDebugLoc(MBBI);
212 // Add callee saved registers to move list.
213 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
217 for (const auto &Info : CSI) {
218 unsigned Reg = Info.getReg();
220 MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
221 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
222 unsigned CFIIndex = MF.addFrameInst(
223 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
224 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
225 .addCFIIndex(CFIIndex)
226 .setMIFlags(MachineInstr::FrameSetup);
230 // Find a scratch register that we can use at the start of the prologue to
231 // re-align the stack pointer. We avoid using callee-save registers since they
232 // may appear to be free when this is called from canUseAsPrologue (during
233 // shrink wrapping), but then no longer be free when this is called from
236 // FIXME: This is a bit conservative, since in the above case we could use one
237 // of the callee-save registers as a scratch temp to re-align the stack pointer,
238 // but we would then have to make sure that we were in fact saving at least one
239 // callee-save register in the prologue, which is additional complexity that
240 // doesn't seem worth the benefit.
241 static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
242 MachineFunction *MF = MBB->getParent();
244 // If MBB is an entry block, use X9 as the scratch register
245 if (&MF->front() == MBB)
248 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
249 LivePhysRegs LiveRegs(&TRI);
250 LiveRegs.addLiveIns(*MBB);
252 // Mark callee saved registers as used so we will not choose them.
253 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
254 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
255 const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(MF);
256 for (unsigned i = 0; CSRegs[i]; ++i)
257 LiveRegs.addReg(CSRegs[i]);
259 // Prefer X9 since it was historically used for the prologue scratch reg.
260 const MachineRegisterInfo &MRI = MF->getRegInfo();
261 if (LiveRegs.available(MRI, AArch64::X9))
264 for (unsigned Reg : AArch64::GPR64RegClass) {
265 if (LiveRegs.available(MRI, Reg))
268 return AArch64::NoRegister;
271 bool AArch64FrameLowering::canUseAsPrologue(
272 const MachineBasicBlock &MBB) const {
273 const MachineFunction *MF = MBB.getParent();
274 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
275 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
276 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
278 // Don't need a scratch register if we're not going to re-align the stack.
279 if (!RegInfo->needsStackRealignment(*MF))
281 // Otherwise, we can use any block as long as it has a scratch register
283 return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
286 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
287 MachineFunction &MF, unsigned StackBumpBytes) const {
288 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
289 const MachineFrameInfo &MFI = MF.getFrameInfo();
290 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
291 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
293 if (AFI->getLocalStackSize() == 0)
296 // 512 is the maximum immediate for stp/ldp that will be used for
297 // callee-save save/restores
298 if (StackBumpBytes >= 512)
301 if (MFI.hasVarSizedObjects())
304 if (RegInfo->needsStackRealignment(MF))
307 // This isn't strictly necessary, but it simplifies things a bit since the
308 // current RedZone handling code assumes the SP is adjusted by the
309 // callee-save save/restore code.
310 if (canUseRedZone(MF))
316 // Convert callee-save register save/restore instruction to do stack pointer
317 // decrement/increment to allocate/deallocate the callee-save stack area by
318 // converting store/load to use pre/post increment version.
319 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
320 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
321 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc) {
324 bool NewIsUnscaled = false;
325 switch (MBBI->getOpcode()) {
327 llvm_unreachable("Unexpected callee-save save/restore opcode!");
329 NewOpc = AArch64::STPXpre;
332 NewOpc = AArch64::STPDpre;
334 case AArch64::STRXui:
335 NewOpc = AArch64::STRXpre;
336 NewIsUnscaled = true;
338 case AArch64::STRDui:
339 NewOpc = AArch64::STRDpre;
340 NewIsUnscaled = true;
343 NewOpc = AArch64::LDPXpost;
346 NewOpc = AArch64::LDPDpost;
348 case AArch64::LDRXui:
349 NewOpc = AArch64::LDRXpost;
350 NewIsUnscaled = true;
352 case AArch64::LDRDui:
353 NewOpc = AArch64::LDRDpost;
354 NewIsUnscaled = true;
358 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
359 MIB.addReg(AArch64::SP, RegState::Define);
361 // Copy all operands other than the immediate offset.
362 unsigned OpndIdx = 0;
363 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
365 MIB.addOperand(MBBI->getOperand(OpndIdx));
367 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
368 "Unexpected immediate offset in first/last callee-save save/restore "
370 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
371 "Unexpected base register in callee-save save/restore instruction!");
372 // Last operand is immediate offset that needs fixing.
373 assert(CSStackSizeInc % 8 == 0);
374 int64_t CSStackSizeIncImm = CSStackSizeInc;
376 CSStackSizeIncImm /= 8;
377 MIB.addImm(CSStackSizeIncImm);
379 MIB.setMIFlags(MBBI->getFlags());
380 MIB.setMemRefs(MBBI->memoperands_begin(), MBBI->memoperands_end());
382 return std::prev(MBB.erase(MBBI));
385 // Fixup callee-save register save/restore instructions to take into account
386 // combined SP bump by adding the local stack size to the stack offsets.
387 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
388 unsigned LocalStackSize) {
389 unsigned Opc = MI.getOpcode();
391 assert((Opc == AArch64::STPXi || Opc == AArch64::STPDi ||
392 Opc == AArch64::STRXui || Opc == AArch64::STRDui ||
393 Opc == AArch64::LDPXi || Opc == AArch64::LDPDi ||
394 Opc == AArch64::LDRXui || Opc == AArch64::LDRDui) &&
395 "Unexpected callee-save save/restore opcode!");
397 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
398 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
399 "Unexpected base register in callee-save save/restore instruction!");
400 // Last operand is immediate offset that needs fixing.
401 MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
402 // All generated opcodes have scaled offsets.
403 assert(LocalStackSize % 8 == 0);
404 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / 8);
407 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
408 MachineBasicBlock &MBB) const {
409 MachineBasicBlock::iterator MBBI = MBB.begin();
410 const MachineFrameInfo &MFI = MF.getFrameInfo();
411 const Function *Fn = MF.getFunction();
412 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
413 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
414 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
415 MachineModuleInfo &MMI = MF.getMMI();
416 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
417 bool needsFrameMoves = MMI.hasDebugInfo() || Fn->needsUnwindTableEntry();
418 bool HasFP = hasFP(MF);
420 // Debug location must be unknown since the first debug location is used
421 // to determine the end of the prologue.
424 // All calls are tail calls in GHC calling conv, and functions have no
425 // prologue/epilogue.
426 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
429 int NumBytes = (int)MFI.getStackSize();
430 if (!AFI->hasStackFrame()) {
431 assert(!HasFP && "unexpected function without stack frame but with FP");
433 // All of the stack allocation is for locals.
434 AFI->setLocalStackSize(NumBytes);
438 // REDZONE: If the stack size is less than 128 bytes, we don't need
439 // to actually allocate.
440 if (canUseRedZone(MF))
441 ++NumRedZoneFunctions;
443 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
444 MachineInstr::FrameSetup);
446 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
447 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
448 // Encode the stack size of the leaf function.
449 unsigned CFIIndex = MF.addFrameInst(
450 MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
451 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
452 .addCFIIndex(CFIIndex)
453 .setMIFlags(MachineInstr::FrameSetup);
458 auto CSStackSize = AFI->getCalleeSavedStackSize();
459 // All of the remaining stack allocations are for locals.
460 AFI->setLocalStackSize(NumBytes - CSStackSize);
462 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
464 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
465 MachineInstr::FrameSetup);
467 } else if (CSStackSize != 0) {
468 MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(MBB, MBBI, DL, TII,
470 NumBytes -= CSStackSize;
472 assert(NumBytes >= 0 && "Negative stack allocation size!?");
474 // Move past the saves of the callee-saved registers, fixing up the offsets
475 // and pre-inc if we decided to combine the callee-save and local stack
476 // pointer bump above.
477 MachineBasicBlock::iterator End = MBB.end();
478 while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
480 fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize());
484 // Only set up FP if we actually need to. Frame pointer is fp = sp - 16.
485 int FPOffset = CSStackSize - 16;
487 FPOffset += AFI->getLocalStackSize();
489 // Issue sub fp, sp, FPOffset or
490 // mov fp,sp when FPOffset is zero.
491 // Note: All stores of callee-saved registers are marked as "FrameSetup".
492 // This code marks the instruction(s) that set the FP also.
493 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
494 MachineInstr::FrameSetup);
497 // Allocate space for the rest of the frame.
499 const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
500 unsigned scratchSPReg = AArch64::SP;
502 if (NeedsRealignment) {
503 scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
504 assert(scratchSPReg != AArch64::NoRegister);
507 // If we're a leaf function, try using the red zone.
508 if (!canUseRedZone(MF))
509 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
510 // the correct value here, as NumBytes also includes padding bytes,
511 // which shouldn't be counted here.
512 emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
513 MachineInstr::FrameSetup);
515 if (NeedsRealignment) {
516 const unsigned Alignment = MFI.getMaxAlignment();
517 const unsigned NrBitsToZero = countTrailingZeros(Alignment);
518 assert(NrBitsToZero > 1);
519 assert(scratchSPReg != AArch64::SP);
521 // SUB X9, SP, NumBytes
522 // -- X9 is temporary register, so shouldn't contain any live data here,
523 // -- free to use. This is already produced by emitFrameOffset above.
524 // AND SP, X9, 0b11111...0000
525 // The logical immediates have a non-trivial encoding. The following
526 // formula computes the encoded immediate with all ones but
527 // NrBitsToZero zero bits as least significant bits.
528 uint32_t andMaskEncoded = (1 << 12) // = N
529 | ((64 - NrBitsToZero) << 6) // immr
530 | ((64 - NrBitsToZero - 1) << 0); // imms
532 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
533 .addReg(scratchSPReg, RegState::Kill)
534 .addImm(andMaskEncoded);
535 AFI->setStackRealigned(true);
539 // If we need a base pointer, set it up here. It's whatever the value of the
540 // stack pointer is at this point. Any variable size objects will be allocated
541 // after this, so we can still use the base pointer to reference locals.
543 // FIXME: Clarify FrameSetup flags here.
544 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
546 if (RegInfo->hasBasePointer(MF)) {
547 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
551 if (needsFrameMoves) {
552 const DataLayout &TD = MF.getDataLayout();
553 const int StackGrowth = -TD.getPointerSize(0);
554 unsigned FramePtr = RegInfo->getFrameRegister(MF);
555 // An example of the prologue:
562 // .cfi_personality 155, ___gxx_personality_v0
564 // .cfi_lsda 16, Lexception33
566 // stp xa,bx, [sp, -#offset]!
568 // stp x28, x27, [sp, #offset-32]
569 // stp fp, lr, [sp, #offset-16]
570 // add fp, sp, #offset - 16
574 // +-------------------------------------------+
575 // 10000 | ........ | ........ | ........ | ........ |
576 // 10004 | ........ | ........ | ........ | ........ |
577 // +-------------------------------------------+
578 // 10008 | ........ | ........ | ........ | ........ |
579 // 1000c | ........ | ........ | ........ | ........ |
580 // +===========================================+
581 // 10010 | X28 Register |
582 // 10014 | X28 Register |
583 // +-------------------------------------------+
584 // 10018 | X27 Register |
585 // 1001c | X27 Register |
586 // +===========================================+
587 // 10020 | Frame Pointer |
588 // 10024 | Frame Pointer |
589 // +-------------------------------------------+
590 // 10028 | Link Register |
591 // 1002c | Link Register |
592 // +===========================================+
593 // 10030 | ........ | ........ | ........ | ........ |
594 // 10034 | ........ | ........ | ........ | ........ |
595 // +-------------------------------------------+
596 // 10038 | ........ | ........ | ........ | ........ |
597 // 1003c | ........ | ........ | ........ | ........ |
598 // +-------------------------------------------+
600 // [sp] = 10030 :: >>initial value<<
601 // sp = 10020 :: stp fp, lr, [sp, #-16]!
602 // fp = sp == 10020 :: mov fp, sp
603 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
604 // sp == 10010 :: >>final value<<
606 // The frame pointer (w29) points to address 10020. If we use an offset of
607 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
608 // for w27, and -32 for w28:
611 // .cfi_def_cfa w29, 16
613 // .cfi_offset w30, -8
615 // .cfi_offset w29, -16
617 // .cfi_offset w27, -24
619 // .cfi_offset w28, -32
622 // Define the current CFA rule to use the provided FP.
623 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
624 unsigned CFIIndex = MF.addFrameInst(
625 MCCFIInstruction::createDefCfa(nullptr, Reg, 2 * StackGrowth));
626 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
627 .addCFIIndex(CFIIndex)
628 .setMIFlags(MachineInstr::FrameSetup);
630 // Encode the stack size of the leaf function.
631 unsigned CFIIndex = MF.addFrameInst(
632 MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
633 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
634 .addCFIIndex(CFIIndex)
635 .setMIFlags(MachineInstr::FrameSetup);
638 // Now emit the moves for whatever callee saved regs we have (including FP,
639 // LR if those are saved).
640 emitCalleeSavedFrameMoves(MBB, MBBI);
644 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
645 MachineBasicBlock &MBB) const {
646 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
647 MachineFrameInfo &MFI = MF.getFrameInfo();
648 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
649 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
651 bool IsTailCallReturn = false;
652 if (MBB.end() != MBBI) {
653 DL = MBBI->getDebugLoc();
654 unsigned RetOpcode = MBBI->getOpcode();
655 IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
656 RetOpcode == AArch64::TCRETURNri;
658 int NumBytes = MFI.getStackSize();
659 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
661 // All calls are tail calls in GHC calling conv, and functions have no
662 // prologue/epilogue.
663 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
666 // Initial and residual are named for consistency with the prologue. Note that
667 // in the epilogue, the residual adjustment is executed first.
668 uint64_t ArgumentPopSize = 0;
669 if (IsTailCallReturn) {
670 MachineOperand &StackAdjust = MBBI->getOperand(1);
672 // For a tail-call in a callee-pops-arguments environment, some or all of
673 // the stack may actually be in use for the call's arguments, this is
674 // calculated during LowerCall and consumed here...
675 ArgumentPopSize = StackAdjust.getImm();
677 // ... otherwise the amount to pop is *all* of the argument space,
678 // conveniently stored in the MachineFunctionInfo by
679 // LowerFormalArguments. This will, of course, be zero for the C calling
681 ArgumentPopSize = AFI->getArgumentStackToRestore();
684 // The stack frame should be like below,
686 // ---------------------- ---
688 // | BytesInStackArgArea| CalleeArgStackSize
689 // | (NumReusableBytes) | (of tail call)
692 // ---------------------| --- |
694 // | CalleeSavedReg | | |
695 // | (CalleeSavedStackSize)| | |
697 // ---------------------| | NumBytes
698 // | | StackSize (StackAdjustUp)
699 // | LocalStackSize | | |
700 // | (covering callee | | |
703 // ---------------------- --- ---
705 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
706 // = StackSize + ArgumentPopSize
708 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
709 // it as the 2nd argument of AArch64ISD::TC_RETURN.
711 auto CSStackSize = AFI->getCalleeSavedStackSize();
712 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
714 if (!CombineSPBump && CSStackSize != 0)
715 convertCalleeSaveRestoreToSPPrePostIncDec(
716 MBB, std::prev(MBB.getFirstTerminator()), DL, TII, CSStackSize);
718 // Move past the restores of the callee-saved registers.
719 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
720 MachineBasicBlock::iterator Begin = MBB.begin();
721 while (LastPopI != Begin) {
723 if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
726 } else if (CombineSPBump)
727 fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize());
730 // If there is a single SP update, insert it before the ret and we're done.
732 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
733 NumBytes + ArgumentPopSize, TII,
734 MachineInstr::FrameDestroy);
738 NumBytes -= CSStackSize;
739 assert(NumBytes >= 0 && "Negative stack allocation size!?");
742 bool RedZone = canUseRedZone(MF);
743 // If this was a redzone leaf function, we don't need to restore the
744 // stack pointer (but we may need to pop stack args for fastcc).
745 if (RedZone && ArgumentPopSize == 0)
748 bool NoCalleeSaveRestore = CSStackSize == 0;
749 int StackRestoreBytes = RedZone ? 0 : NumBytes;
750 if (NoCalleeSaveRestore)
751 StackRestoreBytes += ArgumentPopSize;
752 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
753 StackRestoreBytes, TII, MachineInstr::FrameDestroy);
754 // If we were able to combine the local stack pop with the argument pop,
756 if (NoCalleeSaveRestore || ArgumentPopSize == 0)
761 // Restore the original stack pointer.
762 // FIXME: Rather than doing the math here, we should instead just use
763 // non-post-indexed loads for the restores if we aren't actually going to
764 // be able to save any instructions.
765 if (MFI.hasVarSizedObjects() || AFI->isStackRealigned())
766 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
767 -CSStackSize + 16, TII, MachineInstr::FrameDestroy);
769 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
770 MachineInstr::FrameDestroy);
772 // This must be placed after the callee-save restore code because that code
773 // assumes the SP is at the same location as it was after the callee-save save
774 // code in the prologue.
776 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
777 ArgumentPopSize, TII, MachineInstr::FrameDestroy);
780 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
781 /// debug info. It's the same as what we use for resolving the code-gen
782 /// references for now. FIXME: This can go wrong when references are
783 /// SP-relative and simple call frames aren't used.
784 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
786 unsigned &FrameReg) const {
787 return resolveFrameIndexReference(MF, FI, FrameReg);
790 int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
791 int FI, unsigned &FrameReg,
792 bool PreferFP) const {
793 const MachineFrameInfo &MFI = MF.getFrameInfo();
794 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
795 MF.getSubtarget().getRegisterInfo());
796 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
797 int FPOffset = MFI.getObjectOffset(FI) + 16;
798 int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
799 bool isFixed = MFI.isFixedObjectIndex(FI);
801 // Use frame pointer to reference fixed objects. Use it for locals if
802 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
803 // reliable as a base). Make sure useFPForScavengingIndex() does the
804 // right thing for the emergency spill slot.
806 if (AFI->hasStackFrame()) {
807 // Note: Keeping the following as multiple 'if' statements rather than
808 // merging to a single expression for readability.
810 // Argument access should always use the FP.
813 } else if (hasFP(MF) && !RegInfo->hasBasePointer(MF) &&
814 !RegInfo->needsStackRealignment(MF)) {
815 // Use SP or FP, whichever gives us the best chance of the offset
816 // being in range for direct access. If the FPOffset is positive,
817 // that'll always be best, as the SP will be even further away.
818 // If the FPOffset is negative, we have to keep in mind that the
819 // available offset range for negative offsets is smaller than for
820 // positive ones. If we have variable sized objects, we're stuck with
821 // using the FP regardless, though, as the SP offset is unknown
822 // and we don't have a base pointer available. If an offset is
823 // available via the FP and the SP, use whichever is closest.
824 if (PreferFP || MFI.hasVarSizedObjects() || FPOffset >= 0 ||
825 (FPOffset >= -256 && Offset > -FPOffset))
830 assert((isFixed || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
831 "In the presence of dynamic stack pointer realignment, "
832 "non-argument objects cannot be accessed through the frame pointer");
835 FrameReg = RegInfo->getFrameRegister(MF);
839 // Use the base pointer if we have one.
840 if (RegInfo->hasBasePointer(MF))
841 FrameReg = RegInfo->getBaseRegister();
843 FrameReg = AArch64::SP;
844 // If we're using the red zone for this function, the SP won't actually
845 // be adjusted, so the offsets will be negative. They're also all
846 // within range of the signed 9-bit immediate instructions.
847 if (canUseRedZone(MF))
848 Offset -= AFI->getLocalStackSize();
854 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
855 // Do not set a kill flag on values that are also marked as live-in. This
856 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
857 // callee saved registers.
858 // Omitting the kill flags is conservatively correct even if the live-in
859 // is not used after all.
860 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
861 return getKillRegState(!IsLiveIn);
864 static bool produceCompactUnwindFrame(MachineFunction &MF) {
865 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
866 AttributeSet Attrs = MF.getFunction()->getAttributes();
867 return Subtarget.isTargetMachO() &&
868 !(Subtarget.getTargetLowering()->supportSwiftError() &&
869 Attrs.hasAttrSomewhere(Attribute::SwiftError));
874 RegPairInfo() : Reg1(AArch64::NoRegister), Reg2(AArch64::NoRegister) {}
880 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
882 } // end anonymous namespace
884 static void computeCalleeSaveRegisterPairs(
885 MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
886 const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs) {
891 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
892 MachineFrameInfo &MFI = MF.getFrameInfo();
893 CallingConv::ID CC = MF.getFunction()->getCallingConv();
894 unsigned Count = CSI.size();
896 // MachO's compact unwind format relies on all registers being stored in
898 assert((!produceCompactUnwindFrame(MF) ||
899 CC == CallingConv::PreserveMost ||
901 "Odd number of callee-saved regs to spill!");
902 unsigned Offset = AFI->getCalleeSavedStackSize();
904 for (unsigned i = 0; i < Count; ++i) {
906 RPI.Reg1 = CSI[i].getReg();
908 assert(AArch64::GPR64RegClass.contains(RPI.Reg1) ||
909 AArch64::FPR64RegClass.contains(RPI.Reg1));
910 RPI.IsGPR = AArch64::GPR64RegClass.contains(RPI.Reg1);
912 // Add the next reg to the pair if it is in the same register class.
914 unsigned NextReg = CSI[i + 1].getReg();
915 if ((RPI.IsGPR && AArch64::GPR64RegClass.contains(NextReg)) ||
916 (!RPI.IsGPR && AArch64::FPR64RegClass.contains(NextReg)))
920 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
921 // list to come in sorted by frame index so that we can issue the store
922 // pair instructions directly. Assert if we see anything otherwise.
924 // The order of the registers in the list is controlled by
925 // getCalleeSavedRegs(), so they will always be in-order, as well.
926 assert((!RPI.isPaired() ||
927 (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
928 "Out of order callee saved regs!");
930 // MachO's compact unwind format relies on all registers being stored in
931 // adjacent register pairs.
932 assert((!produceCompactUnwindFrame(MF) ||
933 CC == CallingConv::PreserveMost ||
935 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
936 RPI.Reg1 + 1 == RPI.Reg2))) &&
937 "Callee-save registers not saved as adjacent register pair!");
939 RPI.FrameIdx = CSI[i].getFrameIdx();
941 if (Count * 8 != AFI->getCalleeSavedStackSize() && !RPI.isPaired()) {
942 // Round up size of non-pair to pair size if we need to pad the
943 // callee-save area to ensure 16-byte alignment.
945 assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
946 MFI.setObjectAlignment(RPI.FrameIdx, 16);
947 AFI->setCalleeSaveStackHasFreeSpace(true);
949 Offset -= RPI.isPaired() ? 16 : 8;
950 assert(Offset % 8 == 0);
951 RPI.Offset = Offset / 8;
952 assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
953 "Offset out of bounds for LDP/STP immediate");
955 RegPairs.push_back(RPI);
961 bool AArch64FrameLowering::spillCalleeSavedRegisters(
962 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
963 const std::vector<CalleeSavedInfo> &CSI,
964 const TargetRegisterInfo *TRI) const {
965 MachineFunction &MF = *MBB.getParent();
966 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
968 SmallVector<RegPairInfo, 8> RegPairs;
970 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs);
972 for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
974 RegPairInfo RPI = *RPII;
975 unsigned Reg1 = RPI.Reg1;
976 unsigned Reg2 = RPI.Reg2;
979 // Issue sequence of spills for cs regs. The first spill may be converted
980 // to a pre-decrement store later by emitPrologue if the callee-save stack
981 // area allocation can't be combined with the local stack area allocation.
983 // stp x22, x21, [sp, #0] // addImm(+0)
984 // stp x20, x19, [sp, #16] // addImm(+2)
985 // stp fp, lr, [sp, #32] // addImm(+4)
986 // Rationale: This sequence saves uop updates compared to a sequence of
987 // pre-increment spills like stp xi,xj,[sp,#-16]!
988 // Note: Similar rationale and sequence for restores in epilog.
990 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
992 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
993 DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1);
995 dbgs() << ", " << TRI->getName(Reg2);
996 dbgs() << ") -> fi#(" << RPI.FrameIdx;
998 dbgs() << ", " << RPI.FrameIdx+1;
1001 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
1002 MBB.addLiveIn(Reg1);
1003 if (RPI.isPaired()) {
1004 MBB.addLiveIn(Reg2);
1005 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
1006 MIB.addMemOperand(MF.getMachineMemOperand(
1007 MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
1008 MachineMemOperand::MOStore, 8, 8));
1010 MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
1011 .addReg(AArch64::SP)
1012 .addImm(RPI.Offset) // [sp, #offset*8], where factor*8 is implicit
1013 .setMIFlag(MachineInstr::FrameSetup);
1014 MIB.addMemOperand(MF.getMachineMemOperand(
1015 MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
1016 MachineMemOperand::MOStore, 8, 8));
1021 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
1022 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1023 const std::vector<CalleeSavedInfo> &CSI,
1024 const TargetRegisterInfo *TRI) const {
1025 MachineFunction &MF = *MBB.getParent();
1026 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1028 SmallVector<RegPairInfo, 8> RegPairs;
1030 if (MI != MBB.end())
1031 DL = MI->getDebugLoc();
1033 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs);
1035 for (auto RPII = RegPairs.begin(), RPIE = RegPairs.end(); RPII != RPIE;
1037 RegPairInfo RPI = *RPII;
1038 unsigned Reg1 = RPI.Reg1;
1039 unsigned Reg2 = RPI.Reg2;
1041 // Issue sequence of restores for cs regs. The last restore may be converted
1042 // to a post-increment load later by emitEpilogue if the callee-save stack
1043 // area allocation can't be combined with the local stack area allocation.
1045 // ldp fp, lr, [sp, #32] // addImm(+4)
1046 // ldp x20, x19, [sp, #16] // addImm(+2)
1047 // ldp x22, x21, [sp, #0] // addImm(+0)
1048 // Note: see comment in spillCalleeSavedRegisters()
1051 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
1053 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
1054 DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1);
1056 dbgs() << ", " << TRI->getName(Reg2);
1057 dbgs() << ") -> fi#(" << RPI.FrameIdx;
1059 dbgs() << ", " << RPI.FrameIdx+1;
1062 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
1063 if (RPI.isPaired()) {
1064 MIB.addReg(Reg2, getDefRegState(true));
1065 MIB.addMemOperand(MF.getMachineMemOperand(
1066 MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
1067 MachineMemOperand::MOLoad, 8, 8));
1069 MIB.addReg(Reg1, getDefRegState(true))
1070 .addReg(AArch64::SP)
1071 .addImm(RPI.Offset) // [sp, #offset*8] where the factor*8 is implicit
1072 .setMIFlag(MachineInstr::FrameDestroy);
1073 MIB.addMemOperand(MF.getMachineMemOperand(
1074 MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
1075 MachineMemOperand::MOLoad, 8, 8));
1080 void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
1081 BitVector &SavedRegs,
1082 RegScavenger *RS) const {
1083 // All calls are tail calls in GHC calling conv, and functions have no
1084 // prologue/epilogue.
1085 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
1088 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1089 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1090 MF.getSubtarget().getRegisterInfo());
1091 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1092 unsigned UnspilledCSGPR = AArch64::NoRegister;
1093 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
1095 // The frame record needs to be created by saving the appropriate registers
1097 SavedRegs.set(AArch64::FP);
1098 SavedRegs.set(AArch64::LR);
1101 unsigned BasePointerReg = AArch64::NoRegister;
1102 if (RegInfo->hasBasePointer(MF))
1103 BasePointerReg = RegInfo->getBaseRegister();
1105 bool ExtraCSSpill = false;
1106 const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
1107 // Figure out which callee-saved registers to save/restore.
1108 for (unsigned i = 0; CSRegs[i]; ++i) {
1109 const unsigned Reg = CSRegs[i];
1111 // Add the base pointer register to SavedRegs if it is callee-save.
1112 if (Reg == BasePointerReg)
1115 bool RegUsed = SavedRegs.test(Reg);
1116 unsigned PairedReg = CSRegs[i ^ 1];
1118 if (AArch64::GPR64RegClass.contains(Reg) &&
1119 !RegInfo->isReservedReg(MF, Reg)) {
1120 UnspilledCSGPR = Reg;
1121 UnspilledCSGPRPaired = PairedReg;
1126 // MachO's compact unwind format relies on all registers being stored in
1128 // FIXME: the usual format is actually better if unwinding isn't needed.
1129 if (produceCompactUnwindFrame(MF) && !SavedRegs.test(PairedReg)) {
1130 SavedRegs.set(PairedReg);
1131 if (AArch64::GPR64RegClass.contains(PairedReg) &&
1132 !RegInfo->isReservedReg(MF, PairedReg))
1133 ExtraCSSpill = true;
1137 DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
1138 for (int Reg = SavedRegs.find_first(); Reg != -1;
1139 Reg = SavedRegs.find_next(Reg))
1140 dbgs() << ' ' << PrintReg(Reg, RegInfo);
1143 // If any callee-saved registers are used, the frame cannot be eliminated.
1144 unsigned NumRegsSpilled = SavedRegs.count();
1145 bool CanEliminateFrame = NumRegsSpilled == 0;
1147 // FIXME: Set BigStack if any stack slot references may be out of range.
1148 // For now, just conservatively guestimate based on unscaled indexing
1149 // range. We'll end up allocating an unnecessary spill slot a lot, but
1150 // realistically that's not a big deal at this stage of the game.
1151 // The CSR spill slots have not been allocated yet, so estimateStackSize
1152 // won't include them.
1153 MachineFrameInfo &MFI = MF.getFrameInfo();
1154 unsigned CFSize = MFI.estimateStackSize(MF) + 8 * NumRegsSpilled;
1155 DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n");
1156 bool BigStack = (CFSize >= 256);
1157 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
1158 AFI->setHasStackFrame(true);
1160 // Estimate if we might need to scavenge a register at some point in order
1161 // to materialize a stack offset. If so, either spill one additional
1162 // callee-saved register or reserve a special spill slot to facilitate
1163 // register scavenging. If we already spilled an extra callee-saved register
1164 // above to keep the number of spills even, we don't need to do anything else
1166 if (BigStack && !ExtraCSSpill) {
1167 if (UnspilledCSGPR != AArch64::NoRegister) {
1168 DEBUG(dbgs() << "Spilling " << PrintReg(UnspilledCSGPR, RegInfo)
1169 << " to get a scratch register.\n");
1170 SavedRegs.set(UnspilledCSGPR);
1171 // MachO's compact unwind format relies on all registers being stored in
1172 // pairs, so if we need to spill one extra for BigStack, then we need to
1174 if (produceCompactUnwindFrame(MF))
1175 SavedRegs.set(UnspilledCSGPRPaired);
1176 ExtraCSSpill = true;
1177 NumRegsSpilled = SavedRegs.count();
1180 // If we didn't find an extra callee-saved register to spill, create
1181 // an emergency spill slot.
1182 if (!ExtraCSSpill) {
1183 const TargetRegisterClass *RC = &AArch64::GPR64RegClass;
1184 int FI = MFI.CreateStackObject(RC->getSize(), RC->getAlignment(), false);
1185 RS->addScavengingFrameIndex(FI);
1186 DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
1187 << " as the emergency spill slot.\n");
1191 // Round up to register pair alignment to avoid additional SP adjustment
1193 AFI->setCalleeSavedStackSize(alignTo(8 * NumRegsSpilled, 16));
1196 bool AArch64FrameLowering::enableStackSlotScavenging(
1197 const MachineFunction &MF) const {
1198 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1199 return AFI->hasCalleeSaveStackFreeSpace();