1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
11 // On AArch64, stack frames are structured as follows:
13 // The stack grows downward.
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
19 // At function entry, the "frame" looks as follows:
22 // |-----------------------------------|
24 // | arguments passed on the stack |
26 // |-----------------------------------| <- sp
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
37 // |-----------------------------------|
39 // | arguments passed on the stack |
41 // |-----------------------------------|
43 // | (Win64 only) varargs from reg |
45 // |-----------------------------------|
47 // | callee-saved gpr registers | <--.
48 // | | | On Darwin platforms these
49 // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50 // | | | (frame record first)
51 // | prev_fp, prev_lr | <--'
52 // | (a.k.a. "frame record") |
53 // |-----------------------------------| <- fp(=x29)
55 // | callee-saved fp/simd/SVE regs |
57 // |-----------------------------------|
59 // | SVE stack objects |
61 // |-----------------------------------|
62 // |.empty.space.to.make.part.below....|
63 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
64 // |.the.standard.16-byte.alignment....| compile time; if present)
65 // |-----------------------------------|
67 // | local variables of fixed size |
68 // | including spill slots |
69 // |-----------------------------------| <- bp(not defined by ABI,
70 // |.variable-sized.local.variables....| LLVM chooses X19)
71 // |.(VLAs)............................| (size of this area is unknown at
72 // |...................................| compile time)
73 // |-----------------------------------| <- sp
77 // To access the data in a frame, at-compile time, a constant offset must be
78 // computable from one of the pointers (fp, bp, sp) to access it. The size
79 // of the areas with a dotted background cannot be computed at compile-time
80 // if they are present, making it required to have all three of fp, bp and
81 // sp to be set up to be able to access all contents in the frame areas,
82 // assuming all of the frame areas are non-empty.
84 // For most functions, some of the frame areas are empty. For those functions,
85 // it may not be necessary to set up fp or bp:
86 // * A base pointer is definitely needed when there are both VLAs and local
87 // variables with more-than-default alignment requirements.
88 // * A frame pointer is definitely needed when there are local variables with
89 // more-than-default alignment requirements.
91 // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
92 // callee-saved area, since the unwind encoding does not allow for encoding
93 // this dynamically and existing tools depend on this layout. For other
94 // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
95 // area to allow SVE stack objects (allocated directly below the callee-saves,
96 // if available) to be accessed directly from the framepointer.
97 // The SVE spill/fill instructions have VL-scaled addressing modes such
99 // ldr z8, [fp, #-7 mul vl]
100 // For SVE the size of the vector length (VL) is not known at compile-time, so
101 // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
102 // layout, we don't need to add an unscaled offset to the framepointer before
103 // accessing the SVE object in the frame.
105 // In some cases when a base pointer is not strictly needed, it is generated
106 // anyway when offsets from the frame pointer to access local variables become
107 // so large that the offset can't be encoded in the immediate fields of loads
110 // FIXME: also explain the redzone concept.
111 // FIXME: also explain the concept of reserved call frames.
113 //===----------------------------------------------------------------------===//
115 #include "AArch64FrameLowering.h"
116 #include "AArch64InstrInfo.h"
117 #include "AArch64MachineFunctionInfo.h"
118 #include "AArch64RegisterInfo.h"
119 #include "AArch64StackOffset.h"
120 #include "AArch64Subtarget.h"
121 #include "AArch64TargetMachine.h"
122 #include "MCTargetDesc/AArch64AddressingModes.h"
123 #include "llvm/ADT/ScopeExit.h"
124 #include "llvm/ADT/SmallVector.h"
125 #include "llvm/ADT/Statistic.h"
126 #include "llvm/CodeGen/LivePhysRegs.h"
127 #include "llvm/CodeGen/MachineBasicBlock.h"
128 #include "llvm/CodeGen/MachineFrameInfo.h"
129 #include "llvm/CodeGen/MachineFunction.h"
130 #include "llvm/CodeGen/MachineInstr.h"
131 #include "llvm/CodeGen/MachineInstrBuilder.h"
132 #include "llvm/CodeGen/MachineMemOperand.h"
133 #include "llvm/CodeGen/MachineModuleInfo.h"
134 #include "llvm/CodeGen/MachineOperand.h"
135 #include "llvm/CodeGen/MachineRegisterInfo.h"
136 #include "llvm/CodeGen/RegisterScavenging.h"
137 #include "llvm/CodeGen/TargetInstrInfo.h"
138 #include "llvm/CodeGen/TargetRegisterInfo.h"
139 #include "llvm/CodeGen/TargetSubtargetInfo.h"
140 #include "llvm/CodeGen/WinEHFuncInfo.h"
141 #include "llvm/IR/Attributes.h"
142 #include "llvm/IR/CallingConv.h"
143 #include "llvm/IR/DataLayout.h"
144 #include "llvm/IR/DebugLoc.h"
145 #include "llvm/IR/Function.h"
146 #include "llvm/MC/MCAsmInfo.h"
147 #include "llvm/MC/MCDwarf.h"
148 #include "llvm/Support/CommandLine.h"
149 #include "llvm/Support/Debug.h"
150 #include "llvm/Support/ErrorHandling.h"
151 #include "llvm/Support/MathExtras.h"
152 #include "llvm/Support/raw_ostream.h"
153 #include "llvm/Target/TargetMachine.h"
154 #include "llvm/Target/TargetOptions.h"
160 using namespace llvm;
162 #define DEBUG_TYPE "frame-info"
164 static cl::opt<bool> EnableRedZone("aarch64-redzone",
165 cl::desc("enable use of redzone on AArch64"),
166 cl::init(false), cl::Hidden);
169 ReverseCSRRestoreSeq("reverse-csr-restore-seq",
170 cl::desc("reverse the CSR restore sequence"),
171 cl::init(false), cl::Hidden);
173 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
175 /// This is the biggest offset to the stack pointer we can encode in aarch64
176 /// instructions (without using a separate calculation and a temp register).
177 /// Note that the exception here are vector stores/loads which cannot encode any
178 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
179 static const unsigned DefaultSafeSPDisplacement = 255;
181 /// Look at each instruction that references stack frames and return the stack
182 /// size limit beyond which some of these instructions will require a scratch
183 /// register during their expansion later.
184 static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
185 // FIXME: For now, just conservatively guestimate based on unscaled indexing
186 // range. We'll end up allocating an unnecessary spill slot a lot, but
187 // realistically that's not a big deal at this stage of the game.
188 for (MachineBasicBlock &MBB : MF) {
189 for (MachineInstr &MI : MBB) {
190 if (MI.isDebugInstr() || MI.isPseudo() ||
191 MI.getOpcode() == AArch64::ADDXri ||
192 MI.getOpcode() == AArch64::ADDSXri)
195 for (const MachineOperand &MO : MI.operands()) {
200 if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
201 AArch64FrameOffsetCannotUpdate)
206 return DefaultSafeSPDisplacement;
210 AArch64FrameLowering::getStackIDForScalableVectors() const {
211 return TargetStackID::SVEVector;
214 /// Returns the size of the fixed object area (allocated next to sp on entry)
215 /// On Win64 this may include a var args area and an UnwindHelp object for EH.
216 static unsigned getFixedObjectSize(const MachineFunction &MF,
217 const AArch64FunctionInfo *AFI, bool IsWin64,
219 if (!IsWin64 || IsFunclet) {
220 // Only Win64 uses fixed objects, and then only for the function (not
224 // Var args are stored here in the primary function.
225 const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
226 // To support EH funclets we allocate an UnwindHelp object
227 const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
228 return alignTo(VarArgsArea + UnwindHelpObject, 16);
232 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
233 static StackOffset getSVEStackSize(const MachineFunction &MF) {
234 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
235 return {(int64_t)AFI->getStackSizeSVE(), MVT::nxv1i8};
238 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
241 // Don't use the red zone if the function explicitly asks us not to.
242 // This is typically used for kernel code.
243 if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
246 const MachineFrameInfo &MFI = MF.getFrameInfo();
247 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
248 uint64_t NumBytes = AFI->getLocalStackSize();
250 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128 ||
251 getSVEStackSize(MF));
254 /// hasFP - Return true if the specified function should have a dedicated frame
255 /// pointer register.
256 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
257 const MachineFrameInfo &MFI = MF.getFrameInfo();
258 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
259 // Win64 EH requires a frame pointer if funclets are present, as the locals
260 // are accessed off the frame pointer in both the parent function and the
262 if (MF.hasEHFunclets())
264 // Retain behavior of always omitting the FP for leaf functions when possible.
265 if (MF.getTarget().Options.DisableFramePointerElim(MF))
267 if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
268 MFI.hasStackMap() || MFI.hasPatchPoint() ||
269 RegInfo->needsStackRealignment(MF))
271 // With large callframes around we may need to use FP to access the scavenging
272 // emergency spillslot.
274 // Unfortunately some calls to hasFP() like machine verifier ->
275 // getReservedReg() -> hasFP in the middle of global isel are too early
276 // to know the max call frame size. Hopefully conservatively returning "true"
277 // in those cases is fine.
278 // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
279 if (!MFI.isMaxCallFrameSizeComputed() ||
280 MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
286 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
287 /// not required, we reserve argument space for call sites in the function
288 /// immediately on entry to the current function. This eliminates the need for
289 /// add/sub sp brackets around call sites. Returns true if the call frame is
290 /// included as part of the stack frame.
292 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
293 return !MF.getFrameInfo().hasVarSizedObjects();
296 MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
297 MachineFunction &MF, MachineBasicBlock &MBB,
298 MachineBasicBlock::iterator I) const {
299 const AArch64InstrInfo *TII =
300 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
301 DebugLoc DL = I->getDebugLoc();
302 unsigned Opc = I->getOpcode();
303 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
304 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
306 if (!hasReservedCallFrame(MF)) {
307 unsigned Align = getStackAlignment();
309 int64_t Amount = I->getOperand(0).getImm();
310 Amount = alignTo(Amount, Align);
314 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
315 // doesn't have to pop anything), then the first operand will be zero too so
316 // this adjustment is a no-op.
317 if (CalleePopAmount == 0) {
318 // FIXME: in-function stack adjustment for calls is limited to 24-bits
319 // because there's no guaranteed temporary register available.
321 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
322 // 1) For offset <= 12-bit, we use LSL #0
323 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
324 // LSL #0, and the other uses LSL #12.
326 // Most call frames will be allocated at the start of a function so
327 // this is OK, but it is a limitation that needs dealing with.
328 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
329 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, {Amount, MVT::i8},
332 } else if (CalleePopAmount != 0) {
333 // If the calling convention demands that the callee pops arguments from the
334 // stack, we want to add it back if we have a reserved call frame.
335 assert(CalleePopAmount < 0xffffff && "call frame too large");
336 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
337 {-(int64_t)CalleePopAmount, MVT::i8}, TII);
342 static bool ShouldSignReturnAddress(MachineFunction &MF) {
343 // The function should be signed in the following situations:
344 // - sign-return-address=all
345 // - sign-return-address=non-leaf and the functions spills the LR
347 const Function &F = MF.getFunction();
348 if (!F.hasFnAttribute("sign-return-address"))
351 StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
352 if (Scope.equals("none"))
355 if (Scope.equals("all"))
358 assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
360 for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
361 if (Info.getReg() == AArch64::LR)
367 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
368 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
369 MachineFunction &MF = *MBB.getParent();
370 MachineFrameInfo &MFI = MF.getFrameInfo();
371 const TargetSubtargetInfo &STI = MF.getSubtarget();
372 const MCRegisterInfo *MRI = STI.getRegisterInfo();
373 const TargetInstrInfo *TII = STI.getInstrInfo();
374 DebugLoc DL = MBB.findDebugLoc(MBBI);
376 // Add callee saved registers to move list.
377 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
381 for (const auto &Info : CSI) {
382 unsigned Reg = Info.getReg();
384 MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
385 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
386 unsigned CFIIndex = MF.addFrameInst(
387 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
388 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
389 .addCFIIndex(CFIIndex)
390 .setMIFlags(MachineInstr::FrameSetup);
394 // Find a scratch register that we can use at the start of the prologue to
395 // re-align the stack pointer. We avoid using callee-save registers since they
396 // may appear to be free when this is called from canUseAsPrologue (during
397 // shrink wrapping), but then no longer be free when this is called from
400 // FIXME: This is a bit conservative, since in the above case we could use one
401 // of the callee-save registers as a scratch temp to re-align the stack pointer,
402 // but we would then have to make sure that we were in fact saving at least one
403 // callee-save register in the prologue, which is additional complexity that
404 // doesn't seem worth the benefit.
405 static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
406 MachineFunction *MF = MBB->getParent();
408 // If MBB is an entry block, use X9 as the scratch register
409 if (&MF->front() == MBB)
412 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
413 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
414 LivePhysRegs LiveRegs(TRI);
415 LiveRegs.addLiveIns(*MBB);
417 // Mark callee saved registers as used so we will not choose them.
418 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
419 for (unsigned i = 0; CSRegs[i]; ++i)
420 LiveRegs.addReg(CSRegs[i]);
422 // Prefer X9 since it was historically used for the prologue scratch reg.
423 const MachineRegisterInfo &MRI = MF->getRegInfo();
424 if (LiveRegs.available(MRI, AArch64::X9))
427 for (unsigned Reg : AArch64::GPR64RegClass) {
428 if (LiveRegs.available(MRI, Reg))
431 return AArch64::NoRegister;
434 bool AArch64FrameLowering::canUseAsPrologue(
435 const MachineBasicBlock &MBB) const {
436 const MachineFunction *MF = MBB.getParent();
437 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
438 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
439 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
441 // Don't need a scratch register if we're not going to re-align the stack.
442 if (!RegInfo->needsStackRealignment(*MF))
444 // Otherwise, we can use any block as long as it has a scratch register
446 return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
449 static bool windowsRequiresStackProbe(MachineFunction &MF,
450 uint64_t StackSizeInBytes) {
451 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
452 if (!Subtarget.isTargetWindows())
454 const Function &F = MF.getFunction();
455 // TODO: When implementing stack protectors, take that into account
456 // for the probe threshold.
457 unsigned StackProbeSize = 4096;
458 if (F.hasFnAttribute("stack-probe-size"))
459 F.getFnAttribute("stack-probe-size")
461 .getAsInteger(0, StackProbeSize);
462 return (StackSizeInBytes >= StackProbeSize) &&
463 !F.hasFnAttribute("no-stack-arg-probe");
466 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
467 MachineFunction &MF, uint64_t StackBumpBytes) const {
468 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
469 const MachineFrameInfo &MFI = MF.getFrameInfo();
470 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
471 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
473 if (AFI->getLocalStackSize() == 0)
476 // 512 is the maximum immediate for stp/ldp that will be used for
477 // callee-save save/restores
478 if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
481 if (MFI.hasVarSizedObjects())
484 if (RegInfo->needsStackRealignment(MF))
487 // This isn't strictly necessary, but it simplifies things a bit since the
488 // current RedZone handling code assumes the SP is adjusted by the
489 // callee-save save/restore code.
490 if (canUseRedZone(MF))
493 // When there is an SVE area on the stack, always allocate the
494 // callee-saves and spills/locals separately.
495 if (getSVEStackSize(MF))
501 // Given a load or a store instruction, generate an appropriate unwinding SEH
503 static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
504 const TargetInstrInfo &TII,
505 MachineInstr::MIFlag Flag) {
506 unsigned Opc = MBBI->getOpcode();
507 MachineBasicBlock *MBB = MBBI->getParent();
508 MachineFunction &MF = *MBB->getParent();
509 DebugLoc DL = MBBI->getDebugLoc();
510 unsigned ImmIdx = MBBI->getNumOperands() - 1;
511 int Imm = MBBI->getOperand(ImmIdx).getImm();
512 MachineInstrBuilder MIB;
513 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
514 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
518 llvm_unreachable("No SEH Opcode for this instruction");
519 case AArch64::LDPDpost:
522 case AArch64::STPDpre: {
523 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
524 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
525 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
532 case AArch64::LDPXpost:
535 case AArch64::STPXpre: {
536 Register Reg0 = MBBI->getOperand(1).getReg();
537 Register Reg1 = MBBI->getOperand(2).getReg();
538 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
539 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
543 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
544 .addImm(RegInfo->getSEHRegNum(Reg0))
545 .addImm(RegInfo->getSEHRegNum(Reg1))
550 case AArch64::LDRDpost:
553 case AArch64::STRDpre: {
554 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
555 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
561 case AArch64::LDRXpost:
564 case AArch64::STRXpre: {
565 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
566 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
573 case AArch64::LDPDi: {
574 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
575 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
576 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
584 case AArch64::LDPXi: {
585 Register Reg0 = MBBI->getOperand(0).getReg();
586 Register Reg1 = MBBI->getOperand(1).getReg();
587 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
588 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
592 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
593 .addImm(RegInfo->getSEHRegNum(Reg0))
594 .addImm(RegInfo->getSEHRegNum(Reg1))
599 case AArch64::STRXui:
600 case AArch64::LDRXui: {
601 int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
602 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
608 case AArch64::STRDui:
609 case AArch64::LDRDui: {
610 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
611 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
618 auto I = MBB->insertAfter(MBBI, MIB);
622 // Fix up the SEH opcode associated with the save/restore instruction.
623 static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
624 unsigned LocalStackSize) {
625 MachineOperand *ImmOpnd = nullptr;
626 unsigned ImmIdx = MBBI->getNumOperands() - 1;
627 switch (MBBI->getOpcode()) {
629 llvm_unreachable("Fix the offset in the SEH instruction");
630 case AArch64::SEH_SaveFPLR:
631 case AArch64::SEH_SaveRegP:
632 case AArch64::SEH_SaveReg:
633 case AArch64::SEH_SaveFRegP:
634 case AArch64::SEH_SaveFReg:
635 ImmOpnd = &MBBI->getOperand(ImmIdx);
639 ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
642 // Convert callee-save register save/restore instruction to do stack pointer
643 // decrement/increment to allocate/deallocate the callee-save stack area by
644 // converting store/load to use pre/post increment version.
645 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
646 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
647 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
648 bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
649 // Ignore instructions that do not operate on SP, i.e. shadow call stack
650 // instructions and associated CFI instruction.
651 while (MBBI->getOpcode() == AArch64::STRXpost ||
652 MBBI->getOpcode() == AArch64::LDRXpre ||
653 MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
654 if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
655 assert(MBBI->getOperand(0).getReg() != AArch64::SP);
660 switch (MBBI->getOpcode()) {
662 llvm_unreachable("Unexpected callee-save save/restore opcode!");
664 NewOpc = AArch64::STPXpre;
668 NewOpc = AArch64::STPDpre;
672 NewOpc = AArch64::STPQpre;
675 case AArch64::STRXui:
676 NewOpc = AArch64::STRXpre;
678 case AArch64::STRDui:
679 NewOpc = AArch64::STRDpre;
681 case AArch64::STRQui:
682 NewOpc = AArch64::STRQpre;
685 NewOpc = AArch64::LDPXpost;
689 NewOpc = AArch64::LDPDpost;
693 NewOpc = AArch64::LDPQpost;
696 case AArch64::LDRXui:
697 NewOpc = AArch64::LDRXpost;
699 case AArch64::LDRDui:
700 NewOpc = AArch64::LDRDpost;
702 case AArch64::LDRQui:
703 NewOpc = AArch64::LDRQpost;
706 // Get rid of the SEH code associated with the old instruction.
708 auto SEH = std::next(MBBI);
709 if (AArch64InstrInfo::isSEHInstruction(*SEH))
710 SEH->eraseFromParent();
713 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
714 MIB.addReg(AArch64::SP, RegState::Define);
716 // Copy all operands other than the immediate offset.
717 unsigned OpndIdx = 0;
718 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
720 MIB.add(MBBI->getOperand(OpndIdx));
722 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
723 "Unexpected immediate offset in first/last callee-save save/restore "
725 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
726 "Unexpected base register in callee-save save/restore instruction!");
727 assert(CSStackSizeInc % Scale == 0);
728 MIB.addImm(CSStackSizeInc / Scale);
730 MIB.setMIFlags(MBBI->getFlags());
731 MIB.setMemRefs(MBBI->memoperands());
733 // Generate a new SEH code that corresponds to the new instruction.
736 InsertSEH(*MIB, *TII,
737 InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
740 return std::prev(MBB.erase(MBBI));
743 // Fixup callee-save register save/restore instructions to take into account
744 // combined SP bump by adding the local stack size to the stack offsets.
745 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
746 uint64_t LocalStackSize,
749 if (AArch64InstrInfo::isSEHInstruction(MI))
752 unsigned Opc = MI.getOpcode();
754 // Ignore instructions that do not operate on SP, i.e. shadow call stack
755 // instructions and associated CFI instruction.
756 if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
757 Opc == AArch64::CFI_INSTRUCTION) {
758 if (Opc != AArch64::CFI_INSTRUCTION)
759 assert(MI.getOperand(0).getReg() != AArch64::SP);
766 case AArch64::STRXui:
768 case AArch64::STRDui:
770 case AArch64::LDRXui:
772 case AArch64::LDRDui:
776 case AArch64::STRQui:
778 case AArch64::LDRQui:
782 llvm_unreachable("Unexpected callee-save save/restore opcode!");
785 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
786 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
787 "Unexpected base register in callee-save save/restore instruction!");
788 // Last operand is immediate offset that needs fixing.
789 MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
790 // All generated opcodes have scaled offsets.
791 assert(LocalStackSize % Scale == 0);
792 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
796 auto MBBI = std::next(MachineBasicBlock::iterator(MI));
797 assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
798 assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
799 "Expecting a SEH instruction");
800 fixupSEHOpcode(MBBI, LocalStackSize);
804 static void adaptForLdStOpt(MachineBasicBlock &MBB,
805 MachineBasicBlock::iterator FirstSPPopI,
806 MachineBasicBlock::iterator LastPopI) {
807 // Sometimes (when we restore in the same order as we save), we can end up
808 // with code like this:
810 // ldp x26, x25, [sp]
811 // ldp x24, x23, [sp, #16]
812 // ldp x22, x21, [sp, #32]
813 // ldp x20, x19, [sp, #48]
816 // In this case, it is always better to put the first ldp at the end, so
817 // that the load-store optimizer can run and merge the ldp and the add into
819 // If we managed to grab the first pop instruction, move it to the end.
820 if (ReverseCSRRestoreSeq)
821 MBB.splice(FirstSPPopI, &MBB, LastPopI);
822 // We should end up with something like this now:
824 // ldp x24, x23, [sp, #16]
825 // ldp x22, x21, [sp, #32]
826 // ldp x20, x19, [sp, #48]
827 // ldp x26, x25, [sp]
830 // and the load-store optimizer can merge the last two instructions into:
832 // ldp x26, x25, [sp], #64
836 static bool ShouldSignWithAKey(MachineFunction &MF) {
837 const Function &F = MF.getFunction();
838 if (!F.hasFnAttribute("sign-return-address-key"))
841 const StringRef Key =
842 F.getFnAttribute("sign-return-address-key").getValueAsString();
843 assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
844 return Key.equals_lower("a_key");
847 static bool needsWinCFI(const MachineFunction &MF) {
848 const Function &F = MF.getFunction();
849 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
850 F.needsUnwindTableEntry();
853 static bool isTargetDarwin(const MachineFunction &MF) {
854 return MF.getSubtarget<AArch64Subtarget>().isTargetDarwin();
857 static bool isTargetWindows(const MachineFunction &MF) {
858 return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
861 // Convenience function to determine whether I is an SVE callee save.
862 static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
863 switch (I->getOpcode()) {
866 case AArch64::STR_ZXI:
867 case AArch64::STR_PXI:
868 case AArch64::LDR_ZXI:
869 case AArch64::LDR_PXI:
870 return I->getFlag(MachineInstr::FrameSetup) ||
871 I->getFlag(MachineInstr::FrameDestroy);
875 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
876 MachineBasicBlock &MBB) const {
877 MachineBasicBlock::iterator MBBI = MBB.begin();
878 const MachineFrameInfo &MFI = MF.getFrameInfo();
879 const Function &F = MF.getFunction();
880 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
881 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
882 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
883 MachineModuleInfo &MMI = MF.getMMI();
884 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
885 bool needsFrameMoves =
886 MF.needsFrameMoves() && !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
887 bool HasFP = hasFP(MF);
888 bool NeedsWinCFI = needsWinCFI(MF);
889 bool HasWinCFI = false;
890 auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
892 bool IsFunclet = MBB.isEHFuncletEntry();
894 // At this point, we're going to decide whether or not the function uses a
895 // redzone. In most cases, the function doesn't have a redzone so let's
896 // assume that's false and set it to true in the case that there's a redzone.
897 AFI->setHasRedZone(false);
899 // Debug location must be unknown since the first debug location is used
900 // to determine the end of the prologue.
903 if (ShouldSignReturnAddress(MF)) {
904 if (ShouldSignWithAKey(MF))
905 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
906 .setMIFlag(MachineInstr::FrameSetup);
908 BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
909 .setMIFlag(MachineInstr::FrameSetup);
910 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
911 .setMIFlag(MachineInstr::FrameSetup);
915 MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
916 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
917 .addCFIIndex(CFIIndex)
918 .setMIFlags(MachineInstr::FrameSetup);
921 // All calls are tail calls in GHC calling conv, and functions have no
922 // prologue/epilogue.
923 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
926 // Set tagged base pointer to the bottom of the stack frame.
927 // Ideally it should match SP value after prologue.
928 AFI->setTaggedBasePointerOffset(MFI.getStackSize());
930 const StackOffset &SVEStackSize = getSVEStackSize(MF);
932 // getStackSize() includes all the locals in its size calculation. We don't
933 // include these locals when computing the stack size of a funclet, as they
934 // are allocated in the parent's stack frame and accessed via the frame
935 // pointer from the funclet. We only save the callee saved registers in the
936 // funclet, which are really the callee saved registers of the parent
937 // function, including the funclet.
938 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
939 : MFI.getStackSize();
940 if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
941 assert(!HasFP && "unexpected function without stack frame but with FP");
942 assert(!SVEStackSize &&
943 "unexpected function without stack frame but with SVE objects");
944 // All of the stack allocation is for locals.
945 AFI->setLocalStackSize(NumBytes);
948 // REDZONE: If the stack size is less than 128 bytes, we don't need
949 // to actually allocate.
950 if (canUseRedZone(MF)) {
951 AFI->setHasRedZone(true);
952 ++NumRedZoneFunctions;
954 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
955 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
956 false, NeedsWinCFI, &HasWinCFI);
957 if (!NeedsWinCFI && needsFrameMoves) {
958 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
959 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
960 // Encode the stack size of the leaf function.
961 unsigned CFIIndex = MF.addFrameInst(
962 MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
963 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
964 .addCFIIndex(CFIIndex)
965 .setMIFlags(MachineInstr::FrameSetup);
971 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
972 .setMIFlag(MachineInstr::FrameSetup);
979 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
980 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
982 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
983 // All of the remaining stack allocations are for locals.
984 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
985 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
987 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
988 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
989 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup, false,
990 NeedsWinCFI, &HasWinCFI);
992 } else if (PrologueSaveSize != 0) {
993 MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
994 MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
995 NumBytes -= PrologueSaveSize;
997 assert(NumBytes >= 0 && "Negative stack allocation size!?");
999 // Move past the saves of the callee-saved registers, fixing up the offsets
1000 // and pre-inc if we decided to combine the callee-save and local stack
1001 // pointer bump above.
1002 MachineBasicBlock::iterator End = MBB.end();
1003 while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1004 !IsSVECalleeSave(MBBI)) {
1006 fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
1007 NeedsWinCFI, &HasWinCFI);
1011 // For funclets the FP belongs to the containing function.
1012 if (!IsFunclet && HasFP) {
1013 // Only set up FP if we actually need to.
1014 int64_t FPOffset = isTargetDarwin(MF) ? (AFI->getCalleeSavedStackSize() - 16) : 0;
1017 FPOffset += AFI->getLocalStackSize();
1019 // Issue sub fp, sp, FPOffset or
1020 // mov fp,sp when FPOffset is zero.
1021 // Note: All stores of callee-saved registers are marked as "FrameSetup".
1022 // This code marks the instruction(s) that set the FP also.
1023 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1024 {FPOffset, MVT::i8}, TII, MachineInstr::FrameSetup, false,
1025 NeedsWinCFI, &HasWinCFI);
1028 if (windowsRequiresStackProbe(MF, NumBytes)) {
1029 uint64_t NumWords = NumBytes >> 4;
1032 // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1033 // exceed this amount. We need to move at most 2^24 - 1 into x15.
1034 // This is at most two instructions, MOVZ follwed by MOVK.
1035 // TODO: Fix to use multiple stack alloc unwind codes for stacks
1036 // exceeding 256MB in size.
1037 if (NumBytes >= (1 << 28))
1038 report_fatal_error("Stack size cannot exceed 256MB for stack "
1039 "unwinding purposes");
1041 uint32_t LowNumWords = NumWords & 0xFFFF;
1042 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
1043 .addImm(LowNumWords)
1044 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1045 .setMIFlag(MachineInstr::FrameSetup);
1046 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1047 .setMIFlag(MachineInstr::FrameSetup);
1048 if ((NumWords & 0xFFFF0000) != 0) {
1049 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
1050 .addReg(AArch64::X15)
1051 .addImm((NumWords & 0xFFFF0000) >> 16) // High half
1052 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16))
1053 .setMIFlag(MachineInstr::FrameSetup);
1054 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1055 .setMIFlag(MachineInstr::FrameSetup);
1058 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
1060 .setMIFlags(MachineInstr::FrameSetup);
1063 switch (MF.getTarget().getCodeModel()) {
1064 case CodeModel::Tiny:
1065 case CodeModel::Small:
1066 case CodeModel::Medium:
1067 case CodeModel::Kernel:
1068 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1069 .addExternalSymbol("__chkstk")
1070 .addReg(AArch64::X15, RegState::Implicit)
1071 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1072 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1073 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1074 .setMIFlags(MachineInstr::FrameSetup);
1077 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1078 .setMIFlag(MachineInstr::FrameSetup);
1081 case CodeModel::Large:
1082 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1083 .addReg(AArch64::X16, RegState::Define)
1084 .addExternalSymbol("__chkstk")
1085 .addExternalSymbol("__chkstk")
1086 .setMIFlags(MachineInstr::FrameSetup);
1089 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1090 .setMIFlag(MachineInstr::FrameSetup);
1093 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
1094 .addReg(AArch64::X16, RegState::Kill)
1095 .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1096 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1097 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1098 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1099 .setMIFlags(MachineInstr::FrameSetup);
1102 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1103 .setMIFlag(MachineInstr::FrameSetup);
1108 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1109 .addReg(AArch64::SP, RegState::Kill)
1110 .addReg(AArch64::X15, RegState::Kill)
1111 .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
1112 .setMIFlags(MachineInstr::FrameSetup);
1115 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1117 .setMIFlag(MachineInstr::FrameSetup);
1122 StackOffset AllocateBefore = SVEStackSize, AllocateAfter = {};
1123 MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
1125 // Process the SVE callee-saves to determine what space needs to be
1127 if (AFI->getSVECalleeSavedStackSize()) {
1128 // Find callee save instructions in frame.
1129 CalleeSavesBegin = MBBI;
1130 assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
1131 while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
1133 CalleeSavesEnd = MBBI;
1135 int64_t OffsetToFirstCalleeSaveFromSP =
1136 MFI.getObjectOffset(AFI->getMaxSVECSFrameIndex());
1137 StackOffset OffsetToCalleeSavesFromSP =
1138 StackOffset(OffsetToFirstCalleeSaveFromSP, MVT::nxv1i8) + SVEStackSize;
1139 AllocateBefore -= OffsetToCalleeSavesFromSP;
1140 AllocateAfter = SVEStackSize - AllocateBefore;
1143 // Allocate space for the callee saves (if any).
1144 emitFrameOffset(MBB, CalleeSavesBegin, DL, AArch64::SP, AArch64::SP,
1145 -AllocateBefore, TII,
1146 MachineInstr::FrameSetup);
1148 // Finally allocate remaining SVE stack space.
1149 emitFrameOffset(MBB, CalleeSavesEnd, DL, AArch64::SP, AArch64::SP,
1150 -AllocateAfter, TII,
1151 MachineInstr::FrameSetup);
1153 // Allocate space for the rest of the frame.
1155 // Alignment is required for the parent frame, not the funclet
1156 const bool NeedsRealignment =
1157 !IsFunclet && RegInfo->needsStackRealignment(MF);
1158 unsigned scratchSPReg = AArch64::SP;
1160 if (NeedsRealignment) {
1161 scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1162 assert(scratchSPReg != AArch64::NoRegister);
1165 // If we're a leaf function, try using the red zone.
1166 if (!canUseRedZone(MF))
1167 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1168 // the correct value here, as NumBytes also includes padding bytes,
1169 // which shouldn't be counted here.
1170 emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP,
1171 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
1172 false, NeedsWinCFI, &HasWinCFI);
1174 if (NeedsRealignment) {
1175 const unsigned Alignment = MFI.getMaxAlignment();
1176 const unsigned NrBitsToZero = countTrailingZeros(Alignment);
1177 assert(NrBitsToZero > 1);
1178 assert(scratchSPReg != AArch64::SP);
1180 // SUB X9, SP, NumBytes
1181 // -- X9 is temporary register, so shouldn't contain any live data here,
1182 // -- free to use. This is already produced by emitFrameOffset above.
1183 // AND SP, X9, 0b11111...0000
1184 // The logical immediates have a non-trivial encoding. The following
1185 // formula computes the encoded immediate with all ones but
1186 // NrBitsToZero zero bits as least significant bits.
1187 uint32_t andMaskEncoded = (1 << 12) // = N
1188 | ((64 - NrBitsToZero) << 6) // immr
1189 | ((64 - NrBitsToZero - 1) << 0); // imms
1191 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1192 .addReg(scratchSPReg, RegState::Kill)
1193 .addImm(andMaskEncoded);
1194 AFI->setStackRealigned(true);
1197 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1198 .addImm(NumBytes & andMaskEncoded)
1199 .setMIFlag(MachineInstr::FrameSetup);
1204 // If we need a base pointer, set it up here. It's whatever the value of the
1205 // stack pointer is at this point. Any variable size objects will be allocated
1206 // after this, so we can still use the base pointer to reference locals.
1208 // FIXME: Clarify FrameSetup flags here.
1209 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1211 // For funclets the BP belongs to the containing function.
1212 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
1213 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1217 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1218 .setMIFlag(MachineInstr::FrameSetup);
1222 // The very last FrameSetup instruction indicates the end of prologue. Emit a
1223 // SEH opcode indicating the prologue end.
1224 if (NeedsWinCFI && HasWinCFI) {
1225 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1226 .setMIFlag(MachineInstr::FrameSetup);
1229 // SEH funclets are passed the frame pointer in X1. If the parent
1230 // function uses the base register, then the base register is used
1231 // directly, and is not retrieved from X1.
1232 if (IsFunclet && F.hasPersonalityFn()) {
1233 EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
1234 if (isAsynchronousEHPersonality(Per)) {
1235 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
1236 .addReg(AArch64::X1)
1237 .setMIFlag(MachineInstr::FrameSetup);
1238 MBB.addLiveIn(AArch64::X1);
1242 if (needsFrameMoves) {
1243 const DataLayout &TD = MF.getDataLayout();
1244 const int StackGrowth = isTargetDarwin(MF)
1245 ? (2 * -TD.getPointerSize(0))
1246 : -AFI->getCalleeSavedStackSize();
1247 Register FramePtr = RegInfo->getFrameRegister(MF);
1248 // An example of the prologue:
1255 // .cfi_personality 155, ___gxx_personality_v0
1257 // .cfi_lsda 16, Lexception33
1259 // stp xa,bx, [sp, -#offset]!
1261 // stp x28, x27, [sp, #offset-32]
1262 // stp fp, lr, [sp, #offset-16]
1263 // add fp, sp, #offset - 16
1264 // sub sp, sp, #1360
1267 // +-------------------------------------------+
1268 // 10000 | ........ | ........ | ........ | ........ |
1269 // 10004 | ........ | ........ | ........ | ........ |
1270 // +-------------------------------------------+
1271 // 10008 | ........ | ........ | ........ | ........ |
1272 // 1000c | ........ | ........ | ........ | ........ |
1273 // +===========================================+
1274 // 10010 | X28 Register |
1275 // 10014 | X28 Register |
1276 // +-------------------------------------------+
1277 // 10018 | X27 Register |
1278 // 1001c | X27 Register |
1279 // +===========================================+
1280 // 10020 | Frame Pointer |
1281 // 10024 | Frame Pointer |
1282 // +-------------------------------------------+
1283 // 10028 | Link Register |
1284 // 1002c | Link Register |
1285 // +===========================================+
1286 // 10030 | ........ | ........ | ........ | ........ |
1287 // 10034 | ........ | ........ | ........ | ........ |
1288 // +-------------------------------------------+
1289 // 10038 | ........ | ........ | ........ | ........ |
1290 // 1003c | ........ | ........ | ........ | ........ |
1291 // +-------------------------------------------+
1293 // [sp] = 10030 :: >>initial value<<
1294 // sp = 10020 :: stp fp, lr, [sp, #-16]!
1295 // fp = sp == 10020 :: mov fp, sp
1296 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1297 // sp == 10010 :: >>final value<<
1299 // The frame pointer (w29) points to address 10020. If we use an offset of
1300 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1301 // for w27, and -32 for w28:
1304 // .cfi_def_cfa w29, 16
1306 // .cfi_offset w30, -8
1308 // .cfi_offset w29, -16
1310 // .cfi_offset w27, -24
1312 // .cfi_offset w28, -32
1315 // Define the current CFA rule to use the provided FP.
1316 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1317 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
1318 nullptr, Reg, StackGrowth - FixedObject));
1319 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1320 .addCFIIndex(CFIIndex)
1321 .setMIFlags(MachineInstr::FrameSetup);
1323 // Encode the stack size of the leaf function.
1324 unsigned CFIIndex = MF.addFrameInst(
1325 MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
1326 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1327 .addCFIIndex(CFIIndex)
1328 .setMIFlags(MachineInstr::FrameSetup);
1331 // Now emit the moves for whatever callee saved regs we have (including FP,
1332 // LR if those are saved).
1333 emitCalleeSavedFrameMoves(MBB, MBBI);
1337 static void InsertReturnAddressAuth(MachineFunction &MF,
1338 MachineBasicBlock &MBB) {
1339 if (!ShouldSignReturnAddress(MF))
1341 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1342 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1344 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1346 if (MBBI != MBB.end())
1347 DL = MBBI->getDebugLoc();
1349 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1350 // this instruction can safely used for any v8a architecture.
1351 // From v8.3a onwards there are optimised authenticate LR and return
1352 // instructions, namely RETA{A,B}, that can be used instead.
1353 if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1354 MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1355 BuildMI(MBB, MBBI, DL,
1356 TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1357 .copyImplicitOps(*MBBI);
1362 TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1363 .setMIFlag(MachineInstr::FrameDestroy);
1367 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1368 switch (MI.getOpcode()) {
1371 case AArch64::CATCHRET:
1372 case AArch64::CLEANUPRET:
1377 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
1378 MachineBasicBlock &MBB) const {
1379 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1380 MachineFrameInfo &MFI = MF.getFrameInfo();
1381 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1382 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1384 bool IsTailCallReturn = false;
1385 bool NeedsWinCFI = needsWinCFI(MF);
1386 bool HasWinCFI = false;
1387 bool IsFunclet = false;
1388 auto WinCFI = make_scope_exit([&]() {
1389 if (!MF.hasWinCFI())
1390 MF.setHasWinCFI(HasWinCFI);
1393 if (MBB.end() != MBBI) {
1394 DL = MBBI->getDebugLoc();
1395 unsigned RetOpcode = MBBI->getOpcode();
1396 IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
1397 RetOpcode == AArch64::TCRETURNri ||
1398 RetOpcode == AArch64::TCRETURNriBTI;
1399 IsFunclet = isFuncletReturnInstr(*MBBI);
1402 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1403 : MFI.getStackSize();
1404 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1406 // All calls are tail calls in GHC calling conv, and functions have no
1407 // prologue/epilogue.
1408 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1411 // Initial and residual are named for consistency with the prologue. Note that
1412 // in the epilogue, the residual adjustment is executed first.
1413 uint64_t ArgumentPopSize = 0;
1414 if (IsTailCallReturn) {
1415 MachineOperand &StackAdjust = MBBI->getOperand(1);
1417 // For a tail-call in a callee-pops-arguments environment, some or all of
1418 // the stack may actually be in use for the call's arguments, this is
1419 // calculated during LowerCall and consumed here...
1420 ArgumentPopSize = StackAdjust.getImm();
1422 // ... otherwise the amount to pop is *all* of the argument space,
1423 // conveniently stored in the MachineFunctionInfo by
1424 // LowerFormalArguments. This will, of course, be zero for the C calling
1426 ArgumentPopSize = AFI->getArgumentStackToRestore();
1429 // The stack frame should be like below,
1431 // ---------------------- ---
1433 // | BytesInStackArgArea| CalleeArgStackSize
1434 // | (NumReusableBytes) | (of tail call)
1437 // ---------------------| --- |
1439 // | CalleeSavedReg | | |
1440 // | (CalleeSavedStackSize)| | |
1442 // ---------------------| | NumBytes
1443 // | | StackSize (StackAdjustUp)
1444 // | LocalStackSize | | |
1445 // | (covering callee | | |
1448 // ---------------------- --- ---
1450 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1451 // = StackSize + ArgumentPopSize
1453 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1454 // it as the 2nd argument of AArch64ISD::TC_RETURN.
1456 auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1459 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1460 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1462 uint64_t AfterCSRPopSize = ArgumentPopSize;
1463 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1464 // We cannot rely on the local stack size set in emitPrologue if the function
1465 // has funclets, as funclets have different local stack size requirements, and
1466 // the current value set in emitPrologue may be that of the containing
1468 if (MF.hasEHFunclets())
1469 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1470 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1471 // Assume we can't combine the last pop with the sp restore.
1473 if (!CombineSPBump && PrologueSaveSize != 0) {
1474 MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1475 while (AArch64InstrInfo::isSEHInstruction(*Pop))
1476 Pop = std::prev(Pop);
1477 // Converting the last ldp to a post-index ldp is valid only if the last
1478 // ldp's offset is 0.
1479 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1480 // If the offset is 0, convert it to a post-index ldp.
1481 if (OffsetOp.getImm() == 0)
1482 convertCalleeSaveRestoreToSPPrePostIncDec(
1483 MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
1485 // If not, make sure to emit an add after the last ldp.
1486 // We're doing this by transfering the size to be restored from the
1487 // adjustment *before* the CSR pops to the adjustment *after* the CSR
1489 AfterCSRPopSize += PrologueSaveSize;
1493 // Move past the restores of the callee-saved registers.
1494 // If we plan on combining the sp bump of the local stack size and the callee
1495 // save stack size, we might need to adjust the CSR save and restore offsets.
1496 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
1497 MachineBasicBlock::iterator Begin = MBB.begin();
1498 while (LastPopI != Begin) {
1500 if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
1501 IsSVECalleeSave(LastPopI)) {
1504 } else if (CombineSPBump)
1505 fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
1506 NeedsWinCFI, &HasWinCFI);
1511 BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1512 .setMIFlag(MachineInstr::FrameDestroy);
1515 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1517 // If there is a single SP update, insert it before the ret and we're done.
1518 if (CombineSPBump) {
1519 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1520 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1521 {NumBytes + (int64_t)AfterCSRPopSize, MVT::i8}, TII,
1522 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1523 if (NeedsWinCFI && HasWinCFI)
1524 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1525 TII->get(AArch64::SEH_EpilogEnd))
1526 .setMIFlag(MachineInstr::FrameDestroy);
1530 NumBytes -= PrologueSaveSize;
1531 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1533 // Process the SVE callee-saves to determine what space needs to be
1535 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
1536 MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
1537 if (AFI->getSVECalleeSavedStackSize()) {
1538 RestoreBegin = std::prev(RestoreEnd);;
1539 while (IsSVECalleeSave(RestoreBegin) &&
1540 RestoreBegin != MBB.begin())
1544 assert(IsSVECalleeSave(RestoreBegin) &&
1545 IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
1547 int64_t OffsetToFirstCalleeSaveFromSP =
1548 MFI.getObjectOffset(AFI->getMaxSVECSFrameIndex());
1549 StackOffset OffsetToCalleeSavesFromSP =
1550 StackOffset(OffsetToFirstCalleeSaveFromSP, MVT::nxv1i8) + SVEStackSize;
1551 DeallocateBefore = OffsetToCalleeSavesFromSP;
1552 DeallocateAfter = SVEStackSize - DeallocateBefore;
1555 // Deallocate the SVE area.
1557 if (AFI->isStackRealigned()) {
1558 if (AFI->getSVECalleeSavedStackSize())
1559 // Set SP to start of SVE area, from which the callee-save reloads
1560 // can be done. The code below will deallocate the stack space
1561 // space by moving FP -> SP.
1562 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
1563 -SVEStackSize, TII, MachineInstr::FrameDestroy);
1565 if (AFI->getSVECalleeSavedStackSize()) {
1566 // Deallocate the non-SVE locals first before we can deallocate (and
1567 // restore callee saves) from the SVE area.
1568 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1569 {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy);
1573 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1574 DeallocateBefore, TII, MachineInstr::FrameDestroy);
1576 emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
1577 DeallocateAfter, TII, MachineInstr::FrameDestroy);
1582 bool RedZone = canUseRedZone(MF);
1583 // If this was a redzone leaf function, we don't need to restore the
1584 // stack pointer (but we may need to pop stack args for fastcc).
1585 if (RedZone && AfterCSRPopSize == 0)
1588 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1589 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
1590 if (NoCalleeSaveRestore)
1591 StackRestoreBytes += AfterCSRPopSize;
1593 // If we were able to combine the local stack pop with the argument pop,
1595 bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1597 // If we're done after this, make sure to help the load store optimizer.
1599 adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1601 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1602 {StackRestoreBytes, MVT::i8}, TII,
1603 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1607 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1608 TII->get(AArch64::SEH_EpilogEnd))
1609 .setMIFlag(MachineInstr::FrameDestroy);
1617 // Restore the original stack pointer.
1618 // FIXME: Rather than doing the math here, we should instead just use
1619 // non-post-indexed loads for the restores if we aren't actually going to
1620 // be able to save any instructions.
1621 if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
1622 int64_t OffsetToFrameRecord =
1623 isTargetDarwin(MF) ? (-(int64_t)AFI->getCalleeSavedStackSize() + 16) : 0;
1624 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1625 {OffsetToFrameRecord, MVT::i8},
1626 TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
1627 } else if (NumBytes)
1628 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1629 {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy, false,
1632 // This must be placed after the callee-save restore code because that code
1633 // assumes the SP is at the same location as it was after the callee-save save
1634 // code in the prologue.
1635 if (AfterCSRPopSize) {
1636 // Find an insertion point for the first ldp so that it goes before the
1637 // shadow call stack epilog instruction. This ensures that the restore of
1638 // lr from x18 is placed after the restore from sp.
1639 auto FirstSPPopI = MBB.getFirstTerminator();
1640 while (FirstSPPopI != Begin) {
1641 auto Prev = std::prev(FirstSPPopI);
1642 if (Prev->getOpcode() != AArch64::LDRXpre ||
1643 Prev->getOperand(0).getReg() == AArch64::SP)
1648 adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1650 emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1651 {(int64_t)AfterCSRPopSize, MVT::i8}, TII,
1652 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1654 if (NeedsWinCFI && HasWinCFI)
1655 BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1656 .setMIFlag(MachineInstr::FrameDestroy);
1658 MF.setHasWinCFI(HasWinCFI);
1661 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1662 /// debug info. It's the same as what we use for resolving the code-gen
1663 /// references for now. FIXME: This can go wrong when references are
1664 /// SP-relative and simple call frames aren't used.
1665 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
1667 unsigned &FrameReg) const {
1668 return resolveFrameIndexReference(
1671 MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
1676 int AArch64FrameLowering::getNonLocalFrameIndexReference(
1677 const MachineFunction &MF, int FI) const {
1678 return getSEHFrameIndexOffset(MF, FI);
1681 static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset) {
1682 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1683 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1685 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1687 unsigned FixedObject =
1688 getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
1689 unsigned FPAdjust = isTargetDarwin(MF)
1690 ? 16 : AFI->getCalleeSavedStackSize(MF.getFrameInfo());
1691 return {ObjectOffset + FixedObject + FPAdjust, MVT::i8};
1694 static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset) {
1695 const auto &MFI = MF.getFrameInfo();
1696 return {ObjectOffset + (int64_t)MFI.getStackSize(), MVT::i8};
1699 int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
1701 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1702 MF.getSubtarget().getRegisterInfo());
1703 int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
1704 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1705 ? getFPOffset(MF, ObjectOffset).getBytes()
1706 : getStackOffset(MF, ObjectOffset).getBytes();
1709 StackOffset AArch64FrameLowering::resolveFrameIndexReference(
1710 const MachineFunction &MF, int FI, unsigned &FrameReg, bool PreferFP,
1711 bool ForSimm) const {
1712 const auto &MFI = MF.getFrameInfo();
1713 int64_t ObjectOffset = MFI.getObjectOffset(FI);
1714 bool isFixed = MFI.isFixedObjectIndex(FI);
1715 bool isSVE = MFI.getStackID(FI) == TargetStackID::SVEVector;
1716 return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
1720 StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
1721 const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
1722 unsigned &FrameReg, bool PreferFP, bool ForSimm) const {
1723 const auto &MFI = MF.getFrameInfo();
1724 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1725 MF.getSubtarget().getRegisterInfo());
1726 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1727 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1729 int64_t FPOffset = getFPOffset(MF, ObjectOffset).getBytes();
1730 int64_t Offset = getStackOffset(MF, ObjectOffset).getBytes();
1732 !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1734 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1736 // Use frame pointer to reference fixed objects. Use it for locals if
1737 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1738 // reliable as a base). Make sure useFPForScavengingIndex() does the
1739 // right thing for the emergency spill slot.
1741 if (AFI->hasStackFrame() && !isSVE) {
1742 // We shouldn't prefer using the FP when there is an SVE area
1743 // in between the FP and the non-SVE locals/spills.
1744 PreferFP &= !SVEStackSize;
1746 // Note: Keeping the following as multiple 'if' statements rather than
1747 // merging to a single expression for readability.
1749 // Argument access should always use the FP.
1752 } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1753 // References to the CSR area must use FP if we're re-aligning the stack
1754 // since the dynamically-sized alignment padding is between the SP/BP and
1756 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1758 } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1759 // If the FPOffset is negative and we're producing a signed immediate, we
1760 // have to keep in mind that the available offset range for negative
1761 // offsets is smaller than for positive ones. If an offset is available
1762 // via the FP and the SP, use whichever is closest.
1763 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
1764 PreferFP |= Offset > -FPOffset;
1766 if (MFI.hasVarSizedObjects()) {
1767 // If we have variable sized objects, we can use either FP or BP, as the
1768 // SP offset is unknown. We can use the base pointer if we have one and
1769 // FP is not preferred. If not, we're stuck with using FP.
1770 bool CanUseBP = RegInfo->hasBasePointer(MF);
1771 if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1773 else if (!CanUseBP) { // Can't use BP. Forced to use FP.
1774 assert(!SVEStackSize && "Expected BP to be available");
1777 // else we can use BP and FP, but the offset from FP won't fit.
1778 // That will make us scavenge registers which we can probably avoid by
1779 // using BP. If it won't fit for BP either, we'll scavenge anyway.
1780 } else if (FPOffset >= 0) {
1781 // Use SP or FP, whichever gives us the best chance of the offset
1782 // being in range for direct access. If the FPOffset is positive,
1783 // that'll always be best, as the SP will be even further away.
1785 } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1786 // Funclets access the locals contained in the parent's stack frame
1787 // via the frame pointer, so we have to use the FP in the parent
1791 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1792 "Funclets should only be present on Win64");
1795 // We have the choice between FP and (SP or BP).
1796 if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1802 assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1803 "In the presence of dynamic stack pointer realignment, "
1804 "non-argument/CSR objects cannot be accessed through the frame pointer");
1807 int64_t OffsetToSVEArea =
1808 MFI.getStackSize() - AFI->getCalleeSavedStackSize();
1809 StackOffset FPOffset = {ObjectOffset, MVT::nxv1i8};
1810 StackOffset SPOffset = SVEStackSize +
1811 StackOffset(ObjectOffset, MVT::nxv1i8) +
1812 StackOffset(OffsetToSVEArea, MVT::i8);
1813 // Always use the FP for SVE spills if available and beneficial.
1815 (SPOffset.getBytes() ||
1816 FPOffset.getScalableBytes() < SPOffset.getScalableBytes() ||
1817 RegInfo->needsStackRealignment(MF))) {
1818 FrameReg = RegInfo->getFrameRegister(MF);
1822 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
1823 : (unsigned)AArch64::SP;
1827 StackOffset ScalableOffset = {};
1828 if (UseFP && !(isFixed || isCSR))
1829 ScalableOffset = -SVEStackSize;
1830 if (!UseFP && (isFixed || isCSR))
1831 ScalableOffset = SVEStackSize;
1834 FrameReg = RegInfo->getFrameRegister(MF);
1835 return StackOffset(FPOffset, MVT::i8) + ScalableOffset;
1838 // Use the base pointer if we have one.
1839 if (RegInfo->hasBasePointer(MF))
1840 FrameReg = RegInfo->getBaseRegister();
1842 assert(!MFI.hasVarSizedObjects() &&
1843 "Can't use SP when we have var sized objects.");
1844 FrameReg = AArch64::SP;
1845 // If we're using the red zone for this function, the SP won't actually
1846 // be adjusted, so the offsets will be negative. They're also all
1847 // within range of the signed 9-bit immediate instructions.
1848 if (canUseRedZone(MF))
1849 Offset -= AFI->getLocalStackSize();
1852 return StackOffset(Offset, MVT::i8) + ScalableOffset;
1855 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1856 // Do not set a kill flag on values that are also marked as live-in. This
1857 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1858 // callee saved registers.
1859 // Omitting the kill flags is conservatively correct even if the live-in
1860 // is not used after all.
1861 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1862 return getKillRegState(!IsLiveIn);
1865 static bool produceCompactUnwindFrame(MachineFunction &MF) {
1866 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1867 AttributeList Attrs = MF.getFunction().getAttributes();
1868 return Subtarget.isTargetMachO() &&
1869 !(Subtarget.getTargetLowering()->supportSwiftError() &&
1870 Attrs.hasAttrSomewhere(Attribute::SwiftError));
1873 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
1875 // If we are generating register pairs for a Windows function that requires
1876 // EH support, then pair consecutive registers only. There are no unwind
1877 // opcodes for saves/restores of non-consectuve register pairs.
1878 // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
1879 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1881 // TODO: LR can be paired with any register. We don't support this yet in
1882 // the MCLayer. We need to add support for the save_lrpair unwind code.
1883 if (Reg2 == AArch64::FP)
1887 if (Reg2 == Reg1 + 1)
1892 /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
1893 /// WindowsCFI requires that only consecutive registers can be paired.
1894 /// LR and FP need to be allocated together when the frame needs to save
1895 /// the frame-record. This means any other register pairing with LR is invalid.
1896 static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
1897 bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord) {
1899 return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI);
1901 // If we need to store the frame record, don't pair any register
1902 // with LR other than FP.
1903 if (NeedsFrameRecord)
1904 return Reg2 == AArch64::LR;
1911 struct RegPairInfo {
1912 unsigned Reg1 = AArch64::NoRegister;
1913 unsigned Reg2 = AArch64::NoRegister;
1916 enum RegType { GPR, FPR64, FPR128, PPR, ZPR } Type;
1918 RegPairInfo() = default;
1920 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1922 unsigned getScale() const {
1933 llvm_unreachable("Unsupported type");
1936 bool isScalable() const { return Type == PPR || Type == ZPR; }
1939 } // end anonymous namespace
1941 static void computeCalleeSaveRegisterPairs(
1942 MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
1943 const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
1944 bool &NeedShadowCallStackProlog, bool NeedsFrameRecord) {
1949 bool IsWindows = isTargetWindows(MF);
1950 bool NeedsWinCFI = needsWinCFI(MF);
1951 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1952 MachineFrameInfo &MFI = MF.getFrameInfo();
1953 CallingConv::ID CC = MF.getFunction().getCallingConv();
1954 unsigned Count = CSI.size();
1956 // MachO's compact unwind format relies on all registers being stored in
1958 assert((!produceCompactUnwindFrame(MF) ||
1959 CC == CallingConv::PreserveMost ||
1960 (Count & 1) == 0) &&
1961 "Odd number of callee-saved regs to spill!");
1962 int ByteOffset = AFI->getCalleeSavedStackSize();
1963 int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
1964 // On Linux, we will have either one or zero non-paired register. On Windows
1965 // with CFI, we can have multiple unpaired registers in order to utilize the
1966 // available unwind codes. This flag assures that the alignment fixup is done
1967 // only once, as intened.
1968 bool FixupDone = false;
1969 for (unsigned i = 0; i < Count; ++i) {
1971 RPI.Reg1 = CSI[i].getReg();
1973 if (AArch64::GPR64RegClass.contains(RPI.Reg1))
1974 RPI.Type = RegPairInfo::GPR;
1975 else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
1976 RPI.Type = RegPairInfo::FPR64;
1977 else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
1978 RPI.Type = RegPairInfo::FPR128;
1979 else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
1980 RPI.Type = RegPairInfo::ZPR;
1981 else if (AArch64::PPRRegClass.contains(RPI.Reg1))
1982 RPI.Type = RegPairInfo::PPR;
1984 llvm_unreachable("Unsupported register class.");
1986 // Add the next reg to the pair if it is in the same register class.
1987 if (i + 1 < Count) {
1988 unsigned NextReg = CSI[i + 1].getReg();
1990 case RegPairInfo::GPR:
1991 if (AArch64::GPR64RegClass.contains(NextReg) &&
1992 !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows, NeedsWinCFI,
1996 case RegPairInfo::FPR64:
1997 if (AArch64::FPR64RegClass.contains(NextReg) &&
1998 !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
2001 case RegPairInfo::FPR128:
2002 if (AArch64::FPR128RegClass.contains(NextReg))
2005 case RegPairInfo::PPR:
2006 case RegPairInfo::ZPR:
2011 // If either of the registers to be saved is the lr register, it means that
2012 // we also need to save lr in the shadow call stack.
2013 if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
2014 MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
2015 if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
2016 report_fatal_error("Must reserve x18 to use shadow call stack");
2017 NeedShadowCallStackProlog = true;
2020 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
2021 // list to come in sorted by frame index so that we can issue the store
2022 // pair instructions directly. Assert if we see anything otherwise.
2024 // The order of the registers in the list is controlled by
2025 // getCalleeSavedRegs(), so they will always be in-order, as well.
2026 assert((!RPI.isPaired() ||
2027 (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
2028 "Out of order callee saved regs!");
2030 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2031 RPI.Reg1 == AArch64::LR) &&
2032 "FrameRecord must be allocated together with LR");
2034 // Windows AAPCS has FP and LR reversed.
2035 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2036 RPI.Reg2 == AArch64::LR) &&
2037 "FrameRecord must be allocated together with LR");
2039 // MachO's compact unwind format relies on all registers being stored in
2040 // adjacent register pairs.
2041 assert((!produceCompactUnwindFrame(MF) ||
2042 CC == CallingConv::PreserveMost ||
2044 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2045 RPI.Reg1 + 1 == RPI.Reg2))) &&
2046 "Callee-save registers not saved as adjacent register pair!");
2048 RPI.FrameIdx = CSI[i].getFrameIdx();
2050 int Scale = RPI.getScale();
2051 if (RPI.isScalable())
2052 ScalableByteOffset -= Scale;
2054 ByteOffset -= RPI.isPaired() ? 2 * Scale : Scale;
2056 assert(!(RPI.isScalable() && RPI.isPaired()) &&
2057 "Paired spill/fill instructions don't exist for SVE vectors");
2059 // Round up size of non-pair to pair size if we need to pad the
2060 // callee-save area to ensure 16-byte alignment.
2061 if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
2062 !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2066 assert(ByteOffset % 16 == 0);
2067 assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
2068 MFI.setObjectAlignment(RPI.FrameIdx, 16);
2071 int Offset = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2072 assert(Offset % Scale == 0);
2073 RPI.Offset = Offset / Scale;
2075 assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2076 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2077 "Offset out of bounds for LDP/STP immediate");
2079 RegPairs.push_back(RPI);
2085 bool AArch64FrameLowering::spillCalleeSavedRegisters(
2086 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2087 const std::vector<CalleeSavedInfo> &CSI,
2088 const TargetRegisterInfo *TRI) const {
2089 MachineFunction &MF = *MBB.getParent();
2090 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2091 bool NeedsWinCFI = needsWinCFI(MF);
2093 SmallVector<RegPairInfo, 8> RegPairs;
2095 bool NeedShadowCallStackProlog = false;
2096 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2097 NeedShadowCallStackProlog, hasFP(MF));
2098 const MachineRegisterInfo &MRI = MF.getRegInfo();
2100 if (NeedShadowCallStackProlog) {
2101 // Shadow call stack prolog: str x30, [x18], #8
2102 BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
2103 .addReg(AArch64::X18, RegState::Define)
2104 .addReg(AArch64::LR)
2105 .addReg(AArch64::X18)
2107 .setMIFlag(MachineInstr::FrameSetup);
2110 BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
2111 .setMIFlag(MachineInstr::FrameSetup);
2113 if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
2114 // Emit a CFI instruction that causes 8 to be subtracted from the value of
2115 // x18 when unwinding past this frame.
2116 static const char CFIInst[] = {
2117 dwarf::DW_CFA_val_expression,
2120 static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
2121 static_cast<char>(-8) & 0x7f, // addend (sleb128)
2123 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
2124 nullptr, StringRef(CFIInst, sizeof(CFIInst))));
2125 BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
2126 .addCFIIndex(CFIIndex)
2127 .setMIFlag(MachineInstr::FrameSetup);
2130 // This instruction also makes x18 live-in to the entry block.
2131 MBB.addLiveIn(AArch64::X18);
2134 for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
2136 RegPairInfo RPI = *RPII;
2137 unsigned Reg1 = RPI.Reg1;
2138 unsigned Reg2 = RPI.Reg2;
2141 // Issue sequence of spills for cs regs. The first spill may be converted
2142 // to a pre-decrement store later by emitPrologue if the callee-save stack
2143 // area allocation can't be combined with the local stack area allocation.
2145 // stp x22, x21, [sp, #0] // addImm(+0)
2146 // stp x20, x19, [sp, #16] // addImm(+2)
2147 // stp fp, lr, [sp, #32] // addImm(+4)
2148 // Rationale: This sequence saves uop updates compared to a sequence of
2149 // pre-increment spills like stp xi,xj,[sp,#-16]!
2150 // Note: Similar rationale and sequence for restores in epilog.
2151 unsigned Size, Align;
2153 case RegPairInfo::GPR:
2154 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2158 case RegPairInfo::FPR64:
2159 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2163 case RegPairInfo::FPR128:
2164 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2168 case RegPairInfo::ZPR:
2169 StrOpc = AArch64::STR_ZXI;
2173 case RegPairInfo::PPR:
2174 StrOpc = AArch64::STR_PXI;
2179 LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2180 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2181 dbgs() << ") -> fi#(" << RPI.FrameIdx;
2182 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2185 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2186 "Windows unwdinding requires a consecutive (FP,LR) pair");
2187 // Windows unwind codes require consecutive registers if registers are
2188 // paired. Make the switch here, so that the code below will save (x,x+1)
2190 unsigned FrameIdxReg1 = RPI.FrameIdx;
2191 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2192 if (NeedsWinCFI && RPI.isPaired()) {
2193 std::swap(Reg1, Reg2);
2194 std::swap(FrameIdxReg1, FrameIdxReg2);
2196 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
2197 if (!MRI.isReserved(Reg1))
2198 MBB.addLiveIn(Reg1);
2199 if (RPI.isPaired()) {
2200 if (!MRI.isReserved(Reg2))
2201 MBB.addLiveIn(Reg2);
2202 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
2203 MIB.addMemOperand(MF.getMachineMemOperand(
2204 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2205 MachineMemOperand::MOStore, Size, Align));
2207 MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
2208 .addReg(AArch64::SP)
2209 .addImm(RPI.Offset) // [sp, #offset*scale],
2210 // where factor*scale is implicit
2211 .setMIFlag(MachineInstr::FrameSetup);
2212 MIB.addMemOperand(MF.getMachineMemOperand(
2213 MachinePointerInfo::getFixedStack(MF,FrameIdxReg1),
2214 MachineMemOperand::MOStore, Size, Align));
2216 InsertSEH(MIB, TII, MachineInstr::FrameSetup);
2218 // Update the StackIDs of the SVE stack slots.
2219 MachineFrameInfo &MFI = MF.getFrameInfo();
2220 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
2221 MFI.setStackID(RPI.FrameIdx, TargetStackID::SVEVector);
2227 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
2228 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2229 std::vector<CalleeSavedInfo> &CSI,
2230 const TargetRegisterInfo *TRI) const {
2231 MachineFunction &MF = *MBB.getParent();
2232 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2234 SmallVector<RegPairInfo, 8> RegPairs;
2235 bool NeedsWinCFI = needsWinCFI(MF);
2237 if (MI != MBB.end())
2238 DL = MI->getDebugLoc();
2240 bool NeedShadowCallStackProlog = false;
2241 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2242 NeedShadowCallStackProlog, hasFP(MF));
2244 auto EmitMI = [&](const RegPairInfo &RPI) {
2245 unsigned Reg1 = RPI.Reg1;
2246 unsigned Reg2 = RPI.Reg2;
2248 // Issue sequence of restores for cs regs. The last restore may be converted
2249 // to a post-increment load later by emitEpilogue if the callee-save stack
2250 // area allocation can't be combined with the local stack area allocation.
2252 // ldp fp, lr, [sp, #32] // addImm(+4)
2253 // ldp x20, x19, [sp, #16] // addImm(+2)
2254 // ldp x22, x21, [sp, #0] // addImm(+0)
2255 // Note: see comment in spillCalleeSavedRegisters()
2257 unsigned Size, Align;
2259 case RegPairInfo::GPR:
2260 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2264 case RegPairInfo::FPR64:
2265 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2269 case RegPairInfo::FPR128:
2270 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2274 case RegPairInfo::ZPR:
2275 LdrOpc = AArch64::LDR_ZXI;
2279 case RegPairInfo::PPR:
2280 LdrOpc = AArch64::LDR_PXI;
2285 LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2286 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2287 dbgs() << ") -> fi#(" << RPI.FrameIdx;
2288 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2291 // Windows unwind codes require consecutive registers if registers are
2292 // paired. Make the switch here, so that the code below will save (x,x+1)
2294 unsigned FrameIdxReg1 = RPI.FrameIdx;
2295 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2296 if (NeedsWinCFI && RPI.isPaired()) {
2297 std::swap(Reg1, Reg2);
2298 std::swap(FrameIdxReg1, FrameIdxReg2);
2300 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
2301 if (RPI.isPaired()) {
2302 MIB.addReg(Reg2, getDefRegState(true));
2303 MIB.addMemOperand(MF.getMachineMemOperand(
2304 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2305 MachineMemOperand::MOLoad, Size, Align));
2307 MIB.addReg(Reg1, getDefRegState(true))
2308 .addReg(AArch64::SP)
2309 .addImm(RPI.Offset) // [sp, #offset*scale]
2310 // where factor*scale is implicit
2311 .setMIFlag(MachineInstr::FrameDestroy);
2312 MIB.addMemOperand(MF.getMachineMemOperand(
2313 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2314 MachineMemOperand::MOLoad, Size, Align));
2316 InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
2319 // SVE objects are always restored in reverse order.
2320 for (const RegPairInfo &RPI : reverse(RegPairs))
2321 if (RPI.isScalable())
2324 if (ReverseCSRRestoreSeq) {
2325 for (const RegPairInfo &RPI : reverse(RegPairs))
2326 if (!RPI.isScalable())
2329 for (const RegPairInfo &RPI : RegPairs)
2330 if (!RPI.isScalable())
2333 if (NeedShadowCallStackProlog) {
2334 // Shadow call stack epilog: ldr x30, [x18, #-8]!
2335 BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
2336 .addReg(AArch64::X18, RegState::Define)
2337 .addReg(AArch64::LR, RegState::Define)
2338 .addReg(AArch64::X18)
2340 .setMIFlag(MachineInstr::FrameDestroy);
2346 void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
2347 BitVector &SavedRegs,
2348 RegScavenger *RS) const {
2349 // All calls are tail calls in GHC calling conv, and functions have no
2350 // prologue/epilogue.
2351 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
2354 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2355 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2356 MF.getSubtarget().getRegisterInfo());
2357 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2358 unsigned UnspilledCSGPR = AArch64::NoRegister;
2359 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2361 MachineFrameInfo &MFI = MF.getFrameInfo();
2362 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2364 unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2365 ? RegInfo->getBaseRegister()
2366 : (unsigned)AArch64::NoRegister;
2368 unsigned ExtraCSSpill = 0;
2369 // Figure out which callee-saved registers to save/restore.
2370 for (unsigned i = 0; CSRegs[i]; ++i) {
2371 const unsigned Reg = CSRegs[i];
2373 // Add the base pointer register to SavedRegs if it is callee-save.
2374 if (Reg == BasePointerReg)
2377 bool RegUsed = SavedRegs.test(Reg);
2378 unsigned PairedReg = AArch64::NoRegister;
2379 if (AArch64::GPR64RegClass.contains(Reg) ||
2380 AArch64::FPR64RegClass.contains(Reg) ||
2381 AArch64::FPR128RegClass.contains(Reg))
2382 PairedReg = CSRegs[i ^ 1];
2385 if (AArch64::GPR64RegClass.contains(Reg) &&
2386 !RegInfo->isReservedReg(MF, Reg)) {
2387 UnspilledCSGPR = Reg;
2388 UnspilledCSGPRPaired = PairedReg;
2393 // MachO's compact unwind format relies on all registers being stored in
2395 // FIXME: the usual format is actually better if unwinding isn't needed.
2396 if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2397 !SavedRegs.test(PairedReg)) {
2398 SavedRegs.set(PairedReg);
2399 if (AArch64::GPR64RegClass.contains(PairedReg) &&
2400 !RegInfo->isReservedReg(MF, PairedReg))
2401 ExtraCSSpill = PairedReg;
2405 // Calculates the callee saved stack size.
2406 unsigned CSStackSize = 0;
2407 unsigned SVECSStackSize = 0;
2408 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2409 const MachineRegisterInfo &MRI = MF.getRegInfo();
2410 for (unsigned Reg : SavedRegs.set_bits()) {
2411 auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
2412 if (AArch64::PPRRegClass.contains(Reg) ||
2413 AArch64::ZPRRegClass.contains(Reg))
2414 SVECSStackSize += RegSize;
2416 CSStackSize += RegSize;
2419 // Save number of saved regs, so we can easily update CSStackSize later.
2420 unsigned NumSavedRegs = SavedRegs.count();
2422 // The frame record needs to be created by saving the appropriate registers
2423 uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
2425 windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2426 SavedRegs.set(AArch64::FP);
2427 SavedRegs.set(AArch64::LR);
2430 LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2432 : SavedRegs.set_bits()) dbgs()
2433 << ' ' << printReg(Reg, RegInfo);
2436 // If any callee-saved registers are used, the frame cannot be eliminated.
2437 int64_t SVEStackSize =
2438 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
2439 bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
2441 // The CSR spill slots have not been allocated yet, so estimateStackSize
2442 // won't include them.
2443 unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2445 // Conservatively always assume BigStack when there are SVE spills.
2446 bool BigStack = SVEStackSize ||
2447 (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2448 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2449 AFI->setHasStackFrame(true);
2451 // Estimate if we might need to scavenge a register at some point in order
2452 // to materialize a stack offset. If so, either spill one additional
2453 // callee-saved register or reserve a special spill slot to facilitate
2454 // register scavenging. If we already spilled an extra callee-saved register
2455 // above to keep the number of spills even, we don't need to do anything else
2458 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2459 LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2460 << " to get a scratch register.\n");
2461 SavedRegs.set(UnspilledCSGPR);
2462 // MachO's compact unwind format relies on all registers being stored in
2463 // pairs, so if we need to spill one extra for BigStack, then we need to
2465 if (produceCompactUnwindFrame(MF))
2466 SavedRegs.set(UnspilledCSGPRPaired);
2467 ExtraCSSpill = UnspilledCSGPR;
2470 // If we didn't find an extra callee-saved register to spill, create
2471 // an emergency spill slot.
2472 if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2473 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2474 const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2475 unsigned Size = TRI->getSpillSize(RC);
2476 unsigned Align = TRI->getSpillAlignment(RC);
2477 int FI = MFI.CreateStackObject(Size, Align, false);
2478 RS->addScavengingFrameIndex(FI);
2479 LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2480 << " as the emergency spill slot.\n");
2484 // Adding the size of additional 64bit GPR saves.
2485 CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2486 uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
2487 LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2488 << EstimatedStackSize + AlignedCSStackSize
2491 assert((!MFI.isCalleeSavedInfoValid() ||
2492 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
2493 "Should not invalidate callee saved info");
2495 // Round up to register pair alignment to avoid additional SP adjustment
2497 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2498 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2499 AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
2502 bool AArch64FrameLowering::enableStackSlotScavenging(
2503 const MachineFunction &MF) const {
2504 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2505 return AFI->hasCalleeSaveStackFreeSpace();
2508 /// returns true if there are any SVE callee saves.
2509 static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
2510 int &Min, int &Max) {
2511 Min = std::numeric_limits<int>::max();
2512 Max = std::numeric_limits<int>::min();
2514 if (!MFI.isCalleeSavedInfoValid())
2517 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2518 for (auto &CS : CSI) {
2519 if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
2520 AArch64::PPRRegClass.contains(CS.getReg())) {
2521 assert((Max == std::numeric_limits<int>::min() ||
2522 Max + 1 == CS.getFrameIdx()) &&
2523 "SVE CalleeSaves are not consecutive");
2525 Min = std::min(Min, CS.getFrameIdx());
2526 Max = std::max(Max, CS.getFrameIdx());
2529 return Min != std::numeric_limits<int>::max();
2532 // Process all the SVE stack objects and determine offsets for each
2533 // object. If AssignOffsets is true, the offsets get assigned.
2534 // Fills in the first and last callee-saved frame indices into
2535 // Min/MaxCSFrameIndex, respectively.
2536 // Returns the size of the stack.
2537 static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
2538 int &MinCSFrameIndex,
2539 int &MaxCSFrameIndex,
2540 bool AssignOffsets) {
2541 // First process all fixed stack objects.
2543 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
2544 if (MFI.getStackID(I) == TargetStackID::SVEVector) {
2545 int64_t FixedOffset = -MFI.getObjectOffset(I);
2546 if (FixedOffset > Offset)
2547 Offset = FixedOffset;
2550 auto Assign = [&MFI](int FI, int64_t Offset) {
2551 LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
2552 MFI.setObjectOffset(FI, Offset);
2555 // Then process all callee saved slots.
2556 if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
2557 // Make sure to align the last callee save slot.
2558 MFI.setObjectAlignment(MaxCSFrameIndex, 16U);
2560 // Assign offsets to the callee save slots.
2561 for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
2562 Offset += MFI.getObjectSize(I);
2563 Offset = alignTo(Offset, MFI.getObjectAlignment(I));
2569 // Create a buffer of SVE objects to allocate and sort it.
2570 SmallVector<int, 8> ObjectsToAllocate;
2571 for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
2572 unsigned StackID = MFI.getStackID(I);
2573 if (StackID != TargetStackID::SVEVector)
2575 if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
2577 if (MFI.isDeadObjectIndex(I))
2580 ObjectsToAllocate.push_back(I);
2583 // Allocate all SVE locals and spills
2584 for (unsigned FI : ObjectsToAllocate) {
2585 unsigned Align = MFI.getObjectAlignment(FI);
2586 // FIXME: Given that the length of SVE vectors is not necessarily a power of
2587 // two, we'd need to align every object dynamically at runtime if the
2588 // alignment is larger than 16. This is not yet supported.
2591 "Alignment of scalable vectors > 16 bytes is not yet supported");
2593 Offset = alignTo(Offset + MFI.getObjectSize(FI), Align);
2595 Assign(FI, -Offset);
2601 int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
2602 MachineFrameInfo &MFI) const {
2603 int MinCSFrameIndex, MaxCSFrameIndex;
2604 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
2607 int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
2608 MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
2609 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
2613 void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
2614 MachineFunction &MF, RegScavenger *RS) const {
2615 MachineFrameInfo &MFI = MF.getFrameInfo();
2617 assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
2618 "Upwards growing stack unsupported");
2620 int MinCSFrameIndex, MaxCSFrameIndex;
2621 int64_t SVEStackSize =
2622 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
2624 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2625 AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
2626 AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
2628 // If this function isn't doing Win64-style C++ EH, we don't need to do
2630 if (!MF.hasEHFunclets())
2632 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2633 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2635 MachineBasicBlock &MBB = MF.front();
2636 auto MBBI = MBB.begin();
2637 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2640 // Create an UnwindHelp object.
2641 // The UnwindHelp object is allocated at the start of the fixed object area
2642 int64_t FixedObject =
2643 getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
2644 int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
2645 /*SPOffset*/ -FixedObject,
2646 /*IsImmutable=*/false);
2647 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2649 // We need to store -2 into the UnwindHelp object at the start of the
2652 RS->enterBasicBlockEnd(MBB);
2653 RS->backward(std::prev(MBBI));
2654 unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
2655 assert(DstReg && "There must be a free register after frame setup");
2656 BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2657 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2658 .addReg(DstReg, getKillRegState(true))
2659 .addFrameIndex(UnwindHelpFI)
2663 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP before
2664 /// the update. This is easily retrieved as it is exactly the offset that is set
2665 /// in processFunctionBeforeFrameFinalized.
2666 int AArch64FrameLowering::getFrameIndexReferencePreferSP(
2667 const MachineFunction &MF, int FI, unsigned &FrameReg,
2668 bool IgnoreSPUpdates) const {
2669 const MachineFrameInfo &MFI = MF.getFrameInfo();
2670 if (IgnoreSPUpdates) {
2671 LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
2672 << MFI.getObjectOffset(FI) << "\n");
2673 FrameReg = AArch64::SP;
2674 return MFI.getObjectOffset(FI);
2677 return getFrameIndexReference(MF, FI, FrameReg);
2680 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
2681 /// the parent's frame pointer
2682 unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
2683 const MachineFunction &MF) const {
2687 /// Funclets only need to account for space for the callee saved registers,
2688 /// as the locals are accounted for in the parent's stack frame.
2689 unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
2690 const MachineFunction &MF) const {
2691 // This is the size of the pushed CSRs.
2693 MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
2694 // This is the amount of stack a funclet needs to allocate.
2695 return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
2696 getStackAlignment());