1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the AArch64 implementation of TargetFrameLowering class.
12 // On AArch64, stack frames are structured as follows:
14 // The stack grows downward.
16 // All of the individual frame areas on the frame below are optional, i.e. it's
17 // possible to create a function so that the particular area isn't present
20 // At function entry, the "frame" looks as follows:
23 // |-----------------------------------|
25 // | arguments passed on the stack |
27 // |-----------------------------------| <- sp
31 // After the prologue has run, the frame has the following general structure.
32 // Note that this doesn't depict the case where a red-zone is used. Also,
33 // technically the last frame area (VLAs) doesn't get created until in the
34 // main function body, after the prologue is run. However, it's depicted here
38 // |-----------------------------------|
40 // | arguments passed on the stack |
42 // |-----------------------------------|
44 // | (Win64 only) varargs from reg |
46 // |-----------------------------------|
48 // | prev_fp, prev_lr |
49 // | (a.k.a. "frame record") |
50 // |-----------------------------------| <- fp(=x29)
52 // | other callee-saved registers |
54 // |-----------------------------------|
55 // |.empty.space.to.make.part.below....|
56 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
57 // |.the.standard.16-byte.alignment....| compile time; if present)
58 // |-----------------------------------|
60 // | local variables of fixed size |
61 // | including spill slots |
62 // |-----------------------------------| <- bp(not defined by ABI,
63 // |.variable-sized.local.variables....| LLVM chooses X19)
64 // |.(VLAs)............................| (size of this area is unknown at
65 // |...................................| compile time)
66 // |-----------------------------------| <- sp
70 // To access the data in a frame, at-compile time, a constant offset must be
71 // computable from one of the pointers (fp, bp, sp) to access it. The size
72 // of the areas with a dotted background cannot be computed at compile-time
73 // if they are present, making it required to have all three of fp, bp and
74 // sp to be set up to be able to access all contents in the frame areas,
75 // assuming all of the frame areas are non-empty.
77 // For most functions, some of the frame areas are empty. For those functions,
78 // it may not be necessary to set up fp or bp:
79 // * A base pointer is definitely needed when there are both VLAs and local
80 // variables with more-than-default alignment requirements.
81 // * A frame pointer is definitely needed when there are local variables with
82 // more-than-default alignment requirements.
84 // In some cases when a base pointer is not strictly needed, it is generated
85 // anyway when offsets from the frame pointer to access local variables become
86 // so large that the offset can't be encoded in the immediate fields of loads
89 // FIXME: also explain the redzone concept.
90 // FIXME: also explain the concept of reserved call frames.
92 //===----------------------------------------------------------------------===//
94 #include "AArch64FrameLowering.h"
95 #include "AArch64InstrInfo.h"
96 #include "AArch64MachineFunctionInfo.h"
97 #include "AArch64RegisterInfo.h"
98 #include "AArch64Subtarget.h"
99 #include "AArch64TargetMachine.h"
100 #include "MCTargetDesc/AArch64AddressingModes.h"
101 #include "llvm/ADT/ScopeExit.h"
102 #include "llvm/ADT/SmallVector.h"
103 #include "llvm/ADT/Statistic.h"
104 #include "llvm/CodeGen/LivePhysRegs.h"
105 #include "llvm/CodeGen/MachineBasicBlock.h"
106 #include "llvm/CodeGen/MachineFrameInfo.h"
107 #include "llvm/CodeGen/MachineFunction.h"
108 #include "llvm/CodeGen/MachineInstr.h"
109 #include "llvm/CodeGen/MachineInstrBuilder.h"
110 #include "llvm/CodeGen/MachineMemOperand.h"
111 #include "llvm/CodeGen/MachineModuleInfo.h"
112 #include "llvm/CodeGen/MachineOperand.h"
113 #include "llvm/CodeGen/MachineRegisterInfo.h"
114 #include "llvm/CodeGen/RegisterScavenging.h"
115 #include "llvm/CodeGen/TargetInstrInfo.h"
116 #include "llvm/CodeGen/TargetRegisterInfo.h"
117 #include "llvm/CodeGen/TargetSubtargetInfo.h"
118 #include "llvm/CodeGen/WinEHFuncInfo.h"
119 #include "llvm/IR/Attributes.h"
120 #include "llvm/IR/CallingConv.h"
121 #include "llvm/IR/DataLayout.h"
122 #include "llvm/IR/DebugLoc.h"
123 #include "llvm/IR/Function.h"
124 #include "llvm/MC/MCAsmInfo.h"
125 #include "llvm/MC/MCDwarf.h"
126 #include "llvm/Support/CommandLine.h"
127 #include "llvm/Support/Debug.h"
128 #include "llvm/Support/ErrorHandling.h"
129 #include "llvm/Support/MathExtras.h"
130 #include "llvm/Support/raw_ostream.h"
131 #include "llvm/Target/TargetMachine.h"
132 #include "llvm/Target/TargetOptions.h"
138 using namespace llvm;
140 #define DEBUG_TYPE "frame-info"
142 static cl::opt<bool> EnableRedZone("aarch64-redzone",
143 cl::desc("enable use of redzone on AArch64"),
144 cl::init(false), cl::Hidden);
147 ReverseCSRRestoreSeq("reverse-csr-restore-seq",
148 cl::desc("reverse the CSR restore sequence"),
149 cl::init(false), cl::Hidden);
151 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
153 /// This is the biggest offset to the stack pointer we can encode in aarch64
154 /// instructions (without using a separate calculation and a temp register).
155 /// Note that the exception here are vector stores/loads which cannot encode any
156 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
157 static const unsigned DefaultSafeSPDisplacement = 255;
159 /// Look at each instruction that references stack frames and return the stack
160 /// size limit beyond which some of these instructions will require a scratch
161 /// register during their expansion later.
162 static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
163 // FIXME: For now, just conservatively guestimate based on unscaled indexing
164 // range. We'll end up allocating an unnecessary spill slot a lot, but
165 // realistically that's not a big deal at this stage of the game.
166 for (MachineBasicBlock &MBB : MF) {
167 for (MachineInstr &MI : MBB) {
168 if (MI.isDebugInstr() || MI.isPseudo() ||
169 MI.getOpcode() == AArch64::ADDXri ||
170 MI.getOpcode() == AArch64::ADDSXri)
173 for (const MachineOperand &MO : MI.operands()) {
178 if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
179 AArch64FrameOffsetCannotUpdate)
184 return DefaultSafeSPDisplacement;
187 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
190 // Don't use the red zone if the function explicitly asks us not to.
191 // This is typically used for kernel code.
192 if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
195 const MachineFrameInfo &MFI = MF.getFrameInfo();
196 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
197 unsigned NumBytes = AFI->getLocalStackSize();
199 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
202 /// hasFP - Return true if the specified function should have a dedicated frame
203 /// pointer register.
204 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
205 const MachineFrameInfo &MFI = MF.getFrameInfo();
206 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
207 // Win64 EH requires a frame pointer if funclets are present, as the locals
208 // are accessed off the frame pointer in both the parent function and the
210 if (MF.hasEHFunclets())
212 // Retain behavior of always omitting the FP for leaf functions when possible.
213 if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
215 if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
216 MFI.hasStackMap() || MFI.hasPatchPoint() ||
217 RegInfo->needsStackRealignment(MF))
219 // With large callframes around we may need to use FP to access the scavenging
220 // emergency spillslot.
222 // Unfortunately some calls to hasFP() like machine verifier ->
223 // getReservedReg() -> hasFP in the middle of global isel are too early
224 // to know the max call frame size. Hopefully conservatively returning "true"
225 // in those cases is fine.
226 // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
227 if (!MFI.isMaxCallFrameSizeComputed() ||
228 MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
231 // Win64 SEH requires frame pointer if funclets are present.
232 if (MF.hasLocalEscape())
238 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
239 /// not required, we reserve argument space for call sites in the function
240 /// immediately on entry to the current function. This eliminates the need for
241 /// add/sub sp brackets around call sites. Returns true if the call frame is
242 /// included as part of the stack frame.
244 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
245 return !MF.getFrameInfo().hasVarSizedObjects();
248 MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
249 MachineFunction &MF, MachineBasicBlock &MBB,
250 MachineBasicBlock::iterator I) const {
251 const AArch64InstrInfo *TII =
252 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
253 DebugLoc DL = I->getDebugLoc();
254 unsigned Opc = I->getOpcode();
255 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
256 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
258 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
259 if (!TFI->hasReservedCallFrame(MF)) {
260 unsigned Align = getStackAlignment();
262 int64_t Amount = I->getOperand(0).getImm();
263 Amount = alignTo(Amount, Align);
267 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
268 // doesn't have to pop anything), then the first operand will be zero too so
269 // this adjustment is a no-op.
270 if (CalleePopAmount == 0) {
271 // FIXME: in-function stack adjustment for calls is limited to 24-bits
272 // because there's no guaranteed temporary register available.
274 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
275 // 1) For offset <= 12-bit, we use LSL #0
276 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
277 // LSL #0, and the other uses LSL #12.
279 // Most call frames will be allocated at the start of a function so
280 // this is OK, but it is a limitation that needs dealing with.
281 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
282 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
284 } else if (CalleePopAmount != 0) {
285 // If the calling convention demands that the callee pops arguments from the
286 // stack, we want to add it back if we have a reserved call frame.
287 assert(CalleePopAmount < 0xffffff && "call frame too large");
288 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
294 static bool ShouldSignReturnAddress(MachineFunction &MF) {
295 // The function should be signed in the following situations:
296 // - sign-return-address=all
297 // - sign-return-address=non-leaf and the functions spills the LR
299 const Function &F = MF.getFunction();
300 if (!F.hasFnAttribute("sign-return-address"))
303 StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
304 if (Scope.equals("none"))
307 if (Scope.equals("all"))
310 assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
312 for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
313 if (Info.getReg() == AArch64::LR)
319 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
320 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
321 MachineFunction &MF = *MBB.getParent();
322 MachineFrameInfo &MFI = MF.getFrameInfo();
323 const TargetSubtargetInfo &STI = MF.getSubtarget();
324 const MCRegisterInfo *MRI = STI.getRegisterInfo();
325 const TargetInstrInfo *TII = STI.getInstrInfo();
326 DebugLoc DL = MBB.findDebugLoc(MBBI);
328 // Add callee saved registers to move list.
329 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
333 for (const auto &Info : CSI) {
334 unsigned Reg = Info.getReg();
336 MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
337 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
338 unsigned CFIIndex = MF.addFrameInst(
339 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
340 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
341 .addCFIIndex(CFIIndex)
342 .setMIFlags(MachineInstr::FrameSetup);
346 // Find a scratch register that we can use at the start of the prologue to
347 // re-align the stack pointer. We avoid using callee-save registers since they
348 // may appear to be free when this is called from canUseAsPrologue (during
349 // shrink wrapping), but then no longer be free when this is called from
352 // FIXME: This is a bit conservative, since in the above case we could use one
353 // of the callee-save registers as a scratch temp to re-align the stack pointer,
354 // but we would then have to make sure that we were in fact saving at least one
355 // callee-save register in the prologue, which is additional complexity that
356 // doesn't seem worth the benefit.
357 static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
358 MachineFunction *MF = MBB->getParent();
360 // If MBB is an entry block, use X9 as the scratch register
361 if (&MF->front() == MBB)
364 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
365 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
366 LivePhysRegs LiveRegs(TRI);
367 LiveRegs.addLiveIns(*MBB);
369 // Mark callee saved registers as used so we will not choose them.
370 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
371 for (unsigned i = 0; CSRegs[i]; ++i)
372 LiveRegs.addReg(CSRegs[i]);
374 // Prefer X9 since it was historically used for the prologue scratch reg.
375 const MachineRegisterInfo &MRI = MF->getRegInfo();
376 if (LiveRegs.available(MRI, AArch64::X9))
379 for (unsigned Reg : AArch64::GPR64RegClass) {
380 if (LiveRegs.available(MRI, Reg))
383 return AArch64::NoRegister;
386 bool AArch64FrameLowering::canUseAsPrologue(
387 const MachineBasicBlock &MBB) const {
388 const MachineFunction *MF = MBB.getParent();
389 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
390 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
391 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
393 // Don't need a scratch register if we're not going to re-align the stack.
394 if (!RegInfo->needsStackRealignment(*MF))
396 // Otherwise, we can use any block as long as it has a scratch register
398 return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
401 static bool windowsRequiresStackProbe(MachineFunction &MF,
402 unsigned StackSizeInBytes) {
403 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
404 if (!Subtarget.isTargetWindows())
406 const Function &F = MF.getFunction();
407 // TODO: When implementing stack protectors, take that into account
408 // for the probe threshold.
409 unsigned StackProbeSize = 4096;
410 if (F.hasFnAttribute("stack-probe-size"))
411 F.getFnAttribute("stack-probe-size")
413 .getAsInteger(0, StackProbeSize);
414 return (StackSizeInBytes >= StackProbeSize) &&
415 !F.hasFnAttribute("no-stack-arg-probe");
418 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
419 MachineFunction &MF, unsigned StackBumpBytes) const {
420 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
421 const MachineFrameInfo &MFI = MF.getFrameInfo();
422 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
423 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
425 if (AFI->getLocalStackSize() == 0)
428 // 512 is the maximum immediate for stp/ldp that will be used for
429 // callee-save save/restores
430 if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
433 if (MFI.hasVarSizedObjects())
436 if (RegInfo->needsStackRealignment(MF))
439 // This isn't strictly necessary, but it simplifies things a bit since the
440 // current RedZone handling code assumes the SP is adjusted by the
441 // callee-save save/restore code.
442 if (canUseRedZone(MF))
448 // Given a load or a store instruction, generate an appropriate unwinding SEH
450 static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
451 const TargetInstrInfo &TII,
452 MachineInstr::MIFlag Flag) {
453 unsigned Opc = MBBI->getOpcode();
454 MachineBasicBlock *MBB = MBBI->getParent();
455 MachineFunction &MF = *MBB->getParent();
456 DebugLoc DL = MBBI->getDebugLoc();
457 unsigned ImmIdx = MBBI->getNumOperands() - 1;
458 int Imm = MBBI->getOperand(ImmIdx).getImm();
459 MachineInstrBuilder MIB;
460 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
461 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
465 llvm_unreachable("No SEH Opcode for this instruction");
466 case AArch64::LDPDpost:
469 case AArch64::STPDpre: {
470 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
471 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
472 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
479 case AArch64::LDPXpost:
482 case AArch64::STPXpre: {
483 unsigned Reg0 = MBBI->getOperand(1).getReg();
484 unsigned Reg1 = MBBI->getOperand(2).getReg();
485 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
486 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
490 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
491 .addImm(RegInfo->getSEHRegNum(Reg0))
492 .addImm(RegInfo->getSEHRegNum(Reg1))
497 case AArch64::LDRDpost:
500 case AArch64::STRDpre: {
501 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
502 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
508 case AArch64::LDRXpost:
511 case AArch64::STRXpre: {
512 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
513 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
520 case AArch64::LDPDi: {
521 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
522 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
523 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
531 case AArch64::LDPXi: {
532 unsigned Reg0 = MBBI->getOperand(0).getReg();
533 unsigned Reg1 = MBBI->getOperand(1).getReg();
534 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
535 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
539 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
540 .addImm(RegInfo->getSEHRegNum(Reg0))
541 .addImm(RegInfo->getSEHRegNum(Reg1))
546 case AArch64::STRXui:
547 case AArch64::LDRXui: {
548 int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
549 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
555 case AArch64::STRDui:
556 case AArch64::LDRDui: {
557 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
558 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
565 auto I = MBB->insertAfter(MBBI, MIB);
569 // Fix up the SEH opcode associated with the save/restore instruction.
570 static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
571 unsigned LocalStackSize) {
572 MachineOperand *ImmOpnd = nullptr;
573 unsigned ImmIdx = MBBI->getNumOperands() - 1;
574 switch (MBBI->getOpcode()) {
576 llvm_unreachable("Fix the offset in the SEH instruction");
577 case AArch64::SEH_SaveFPLR:
578 case AArch64::SEH_SaveRegP:
579 case AArch64::SEH_SaveReg:
580 case AArch64::SEH_SaveFRegP:
581 case AArch64::SEH_SaveFReg:
582 ImmOpnd = &MBBI->getOperand(ImmIdx);
586 ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
589 // Convert callee-save register save/restore instruction to do stack pointer
590 // decrement/increment to allocate/deallocate the callee-save stack area by
591 // converting store/load to use pre/post increment version.
592 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
593 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
594 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
595 bool NeedsWinCFI, bool InProlog = true) {
596 // Ignore instructions that do not operate on SP, i.e. shadow call stack
597 // instructions and associated CFI instruction.
598 while (MBBI->getOpcode() == AArch64::STRXpost ||
599 MBBI->getOpcode() == AArch64::LDRXpre ||
600 MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
601 if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
602 assert(MBBI->getOperand(0).getReg() != AArch64::SP);
607 switch (MBBI->getOpcode()) {
609 llvm_unreachable("Unexpected callee-save save/restore opcode!");
611 NewOpc = AArch64::STPXpre;
615 NewOpc = AArch64::STPDpre;
619 NewOpc = AArch64::STPQpre;
622 case AArch64::STRXui:
623 NewOpc = AArch64::STRXpre;
625 case AArch64::STRDui:
626 NewOpc = AArch64::STRDpre;
628 case AArch64::STRQui:
629 NewOpc = AArch64::STRQpre;
632 NewOpc = AArch64::LDPXpost;
636 NewOpc = AArch64::LDPDpost;
640 NewOpc = AArch64::LDPQpost;
643 case AArch64::LDRXui:
644 NewOpc = AArch64::LDRXpost;
646 case AArch64::LDRDui:
647 NewOpc = AArch64::LDRDpost;
649 case AArch64::LDRQui:
650 NewOpc = AArch64::LDRQpost;
653 // Get rid of the SEH code associated with the old instruction.
655 auto SEH = std::next(MBBI);
656 if (AArch64InstrInfo::isSEHInstruction(*SEH))
657 SEH->eraseFromParent();
660 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
661 MIB.addReg(AArch64::SP, RegState::Define);
663 // Copy all operands other than the immediate offset.
664 unsigned OpndIdx = 0;
665 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
667 MIB.add(MBBI->getOperand(OpndIdx));
669 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
670 "Unexpected immediate offset in first/last callee-save save/restore "
672 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
673 "Unexpected base register in callee-save save/restore instruction!");
674 assert(CSStackSizeInc % Scale == 0);
675 MIB.addImm(CSStackSizeInc / Scale);
677 MIB.setMIFlags(MBBI->getFlags());
678 MIB.setMemRefs(MBBI->memoperands());
680 // Generate a new SEH code that corresponds to the new instruction.
682 InsertSEH(*MIB, *TII,
683 InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
685 return std::prev(MBB.erase(MBBI));
688 // Fixup callee-save register save/restore instructions to take into account
689 // combined SP bump by adding the local stack size to the stack offsets.
690 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
691 unsigned LocalStackSize,
693 if (AArch64InstrInfo::isSEHInstruction(MI))
696 unsigned Opc = MI.getOpcode();
698 // Ignore instructions that do not operate on SP, i.e. shadow call stack
699 // instructions and associated CFI instruction.
700 if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
701 Opc == AArch64::CFI_INSTRUCTION) {
702 if (Opc != AArch64::CFI_INSTRUCTION)
703 assert(MI.getOperand(0).getReg() != AArch64::SP);
710 case AArch64::STRXui:
712 case AArch64::STRDui:
714 case AArch64::LDRXui:
716 case AArch64::LDRDui:
720 case AArch64::STRQui:
722 case AArch64::LDRQui:
726 llvm_unreachable("Unexpected callee-save save/restore opcode!");
729 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
730 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
731 "Unexpected base register in callee-save save/restore instruction!");
732 // Last operand is immediate offset that needs fixing.
733 MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
734 // All generated opcodes have scaled offsets.
735 assert(LocalStackSize % Scale == 0);
736 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
739 auto MBBI = std::next(MachineBasicBlock::iterator(MI));
740 assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
741 assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
742 "Expecting a SEH instruction");
743 fixupSEHOpcode(MBBI, LocalStackSize);
747 static void adaptForLdStOpt(MachineBasicBlock &MBB,
748 MachineBasicBlock::iterator FirstSPPopI,
749 MachineBasicBlock::iterator LastPopI) {
750 // Sometimes (when we restore in the same order as we save), we can end up
751 // with code like this:
753 // ldp x26, x25, [sp]
754 // ldp x24, x23, [sp, #16]
755 // ldp x22, x21, [sp, #32]
756 // ldp x20, x19, [sp, #48]
759 // In this case, it is always better to put the first ldp at the end, so
760 // that the load-store optimizer can run and merge the ldp and the add into
762 // If we managed to grab the first pop instruction, move it to the end.
763 if (ReverseCSRRestoreSeq)
764 MBB.splice(FirstSPPopI, &MBB, LastPopI);
765 // We should end up with something like this now:
767 // ldp x24, x23, [sp, #16]
768 // ldp x22, x21, [sp, #32]
769 // ldp x20, x19, [sp, #48]
770 // ldp x26, x25, [sp]
773 // and the load-store optimizer can merge the last two instructions into:
775 // ldp x26, x25, [sp], #64
779 static bool ShouldSignWithAKey(MachineFunction &MF) {
780 const Function &F = MF.getFunction();
781 if (!F.hasFnAttribute("sign-return-address-key"))
784 const StringRef Key =
785 F.getFnAttribute("sign-return-address-key").getValueAsString();
786 assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
787 return Key.equals_lower("a_key");
790 static bool needsWinCFI(const MachineFunction &MF) {
791 const Function &F = MF.getFunction();
792 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
793 F.needsUnwindTableEntry();
796 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
797 MachineBasicBlock &MBB) const {
798 MachineBasicBlock::iterator MBBI = MBB.begin();
799 const MachineFrameInfo &MFI = MF.getFrameInfo();
800 const Function &F = MF.getFunction();
801 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
802 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
803 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
804 MachineModuleInfo &MMI = MF.getMMI();
805 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
806 bool needsFrameMoves = (MMI.hasDebugInfo() || F.needsUnwindTableEntry()) &&
807 !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
808 bool HasFP = hasFP(MF);
809 bool NeedsWinCFI = needsWinCFI(MF);
810 MF.setHasWinCFI(NeedsWinCFI);
811 bool IsFunclet = MBB.isEHFuncletEntry();
813 // At this point, we're going to decide whether or not the function uses a
814 // redzone. In most cases, the function doesn't have a redzone so let's
815 // assume that's false and set it to true in the case that there's a redzone.
816 AFI->setHasRedZone(false);
818 // Debug location must be unknown since the first debug location is used
819 // to determine the end of the prologue.
822 if (ShouldSignReturnAddress(MF)) {
823 if (ShouldSignWithAKey(MF))
824 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
825 .setMIFlag(MachineInstr::FrameSetup);
827 BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
828 .setMIFlag(MachineInstr::FrameSetup);
829 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
830 .setMIFlag(MachineInstr::FrameSetup);
834 MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
835 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
836 .addCFIIndex(CFIIndex)
837 .setMIFlags(MachineInstr::FrameSetup);
840 // All calls are tail calls in GHC calling conv, and functions have no
841 // prologue/epilogue.
842 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
845 // getStackSize() includes all the locals in its size calculation. We don't
846 // include these locals when computing the stack size of a funclet, as they
847 // are allocated in the parent's stack frame and accessed via the frame
848 // pointer from the funclet. We only save the callee saved registers in the
849 // funclet, which are really the callee saved registers of the parent
850 // function, including the funclet.
851 int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
852 : (int)MFI.getStackSize();
853 if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
854 assert(!HasFP && "unexpected function without stack frame but with FP");
855 // All of the stack allocation is for locals.
856 AFI->setLocalStackSize(NumBytes);
859 // REDZONE: If the stack size is less than 128 bytes, we don't need
860 // to actually allocate.
861 if (canUseRedZone(MF)) {
862 AFI->setHasRedZone(true);
863 ++NumRedZoneFunctions;
865 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
866 MachineInstr::FrameSetup, false, NeedsWinCFI);
868 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
869 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
870 // Encode the stack size of the leaf function.
871 unsigned CFIIndex = MF.addFrameInst(
872 MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
873 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
874 .addCFIIndex(CFIIndex)
875 .setMIFlags(MachineInstr::FrameSetup);
880 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
881 .setMIFlag(MachineInstr::FrameSetup);
887 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
888 // Var args are accounted for in the containing function, so don't
889 // include them for funclets.
890 unsigned FixedObject = (IsWin64 && !IsFunclet) ?
891 alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
893 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
894 // All of the remaining stack allocations are for locals.
895 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
896 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
898 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
899 MachineInstr::FrameSetup, false, NeedsWinCFI);
901 } else if (PrologueSaveSize != 0) {
902 MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
903 MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI);
904 NumBytes -= PrologueSaveSize;
906 assert(NumBytes >= 0 && "Negative stack allocation size!?");
908 // Move past the saves of the callee-saved registers, fixing up the offsets
909 // and pre-inc if we decided to combine the callee-save and local stack
910 // pointer bump above.
911 MachineBasicBlock::iterator End = MBB.end();
912 while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
914 fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
919 // The code below is not applicable to funclets. We have emitted all the SEH
920 // opcodes that we needed to emit. The FP and BP belong to the containing
924 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
925 .setMIFlag(MachineInstr::FrameSetup);
930 // Only set up FP if we actually need to. Frame pointer is fp =
931 // sp - fixedobject - 16.
932 int FPOffset = AFI->getCalleeSavedStackSize() - 16;
934 FPOffset += AFI->getLocalStackSize();
936 // Issue sub fp, sp, FPOffset or
937 // mov fp,sp when FPOffset is zero.
938 // Note: All stores of callee-saved registers are marked as "FrameSetup".
939 // This code marks the instruction(s) that set the FP also.
940 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
941 MachineInstr::FrameSetup, false, NeedsWinCFI);
944 if (windowsRequiresStackProbe(MF, NumBytes)) {
945 uint32_t NumWords = NumBytes >> 4;
947 // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
948 // exceed this amount. We need to move at most 2^24 - 1 into x15.
949 // This is at most two instructions, MOVZ follwed by MOVK.
950 // TODO: Fix to use multiple stack alloc unwind codes for stacks
951 // exceeding 256MB in size.
952 if (NumBytes >= (1 << 28))
953 report_fatal_error("Stack size cannot exceed 256MB for stack "
954 "unwinding purposes");
956 uint32_t LowNumWords = NumWords & 0xFFFF;
957 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
959 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
960 .setMIFlag(MachineInstr::FrameSetup);
961 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
962 .setMIFlag(MachineInstr::FrameSetup);
963 if ((NumWords & 0xFFFF0000) != 0) {
964 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
965 .addReg(AArch64::X15)
966 .addImm((NumWords & 0xFFFF0000) >> 16) // High half
967 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16))
968 .setMIFlag(MachineInstr::FrameSetup);
969 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
970 .setMIFlag(MachineInstr::FrameSetup);
973 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
975 .setMIFlags(MachineInstr::FrameSetup);
978 switch (MF.getTarget().getCodeModel()) {
979 case CodeModel::Tiny:
980 case CodeModel::Small:
981 case CodeModel::Medium:
982 case CodeModel::Kernel:
983 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
984 .addExternalSymbol("__chkstk")
985 .addReg(AArch64::X15, RegState::Implicit)
986 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
987 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
988 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
989 .setMIFlags(MachineInstr::FrameSetup);
991 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
992 .setMIFlag(MachineInstr::FrameSetup);
994 case CodeModel::Large:
995 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
996 .addReg(AArch64::X16, RegState::Define)
997 .addExternalSymbol("__chkstk")
998 .addExternalSymbol("__chkstk")
999 .setMIFlags(MachineInstr::FrameSetup);
1001 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1002 .setMIFlag(MachineInstr::FrameSetup);
1004 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
1005 .addReg(AArch64::X16, RegState::Kill)
1006 .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1007 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1008 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1009 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1010 .setMIFlags(MachineInstr::FrameSetup);
1012 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1013 .setMIFlag(MachineInstr::FrameSetup);
1017 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1018 .addReg(AArch64::SP, RegState::Kill)
1019 .addReg(AArch64::X15, RegState::Kill)
1020 .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
1021 .setMIFlags(MachineInstr::FrameSetup);
1023 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1025 .setMIFlag(MachineInstr::FrameSetup);
1029 // Allocate space for the rest of the frame.
1031 const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
1032 unsigned scratchSPReg = AArch64::SP;
1034 if (NeedsRealignment) {
1035 scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1036 assert(scratchSPReg != AArch64::NoRegister);
1039 // If we're a leaf function, try using the red zone.
1040 if (!canUseRedZone(MF))
1041 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1042 // the correct value here, as NumBytes also includes padding bytes,
1043 // which shouldn't be counted here.
1044 emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
1045 MachineInstr::FrameSetup, false, NeedsWinCFI);
1047 if (NeedsRealignment) {
1048 const unsigned Alignment = MFI.getMaxAlignment();
1049 const unsigned NrBitsToZero = countTrailingZeros(Alignment);
1050 assert(NrBitsToZero > 1);
1051 assert(scratchSPReg != AArch64::SP);
1053 // SUB X9, SP, NumBytes
1054 // -- X9 is temporary register, so shouldn't contain any live data here,
1055 // -- free to use. This is already produced by emitFrameOffset above.
1056 // AND SP, X9, 0b11111...0000
1057 // The logical immediates have a non-trivial encoding. The following
1058 // formula computes the encoded immediate with all ones but
1059 // NrBitsToZero zero bits as least significant bits.
1060 uint32_t andMaskEncoded = (1 << 12) // = N
1061 | ((64 - NrBitsToZero) << 6) // immr
1062 | ((64 - NrBitsToZero - 1) << 0); // imms
1064 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1065 .addReg(scratchSPReg, RegState::Kill)
1066 .addImm(andMaskEncoded);
1067 AFI->setStackRealigned(true);
1069 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1070 .addImm(NumBytes & andMaskEncoded)
1071 .setMIFlag(MachineInstr::FrameSetup);
1075 // If we need a base pointer, set it up here. It's whatever the value of the
1076 // stack pointer is at this point. Any variable size objects will be allocated
1077 // after this, so we can still use the base pointer to reference locals.
1079 // FIXME: Clarify FrameSetup flags here.
1080 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1082 if (RegInfo->hasBasePointer(MF)) {
1083 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1086 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1087 .setMIFlag(MachineInstr::FrameSetup);
1090 // The very last FrameSetup instruction indicates the end of prologue. Emit a
1091 // SEH opcode indicating the prologue end.
1093 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1094 .setMIFlag(MachineInstr::FrameSetup);
1096 if (needsFrameMoves) {
1097 const DataLayout &TD = MF.getDataLayout();
1098 const int StackGrowth = -TD.getPointerSize(0);
1099 unsigned FramePtr = RegInfo->getFrameRegister(MF);
1100 // An example of the prologue:
1107 // .cfi_personality 155, ___gxx_personality_v0
1109 // .cfi_lsda 16, Lexception33
1111 // stp xa,bx, [sp, -#offset]!
1113 // stp x28, x27, [sp, #offset-32]
1114 // stp fp, lr, [sp, #offset-16]
1115 // add fp, sp, #offset - 16
1116 // sub sp, sp, #1360
1119 // +-------------------------------------------+
1120 // 10000 | ........ | ........ | ........ | ........ |
1121 // 10004 | ........ | ........ | ........ | ........ |
1122 // +-------------------------------------------+
1123 // 10008 | ........ | ........ | ........ | ........ |
1124 // 1000c | ........ | ........ | ........ | ........ |
1125 // +===========================================+
1126 // 10010 | X28 Register |
1127 // 10014 | X28 Register |
1128 // +-------------------------------------------+
1129 // 10018 | X27 Register |
1130 // 1001c | X27 Register |
1131 // +===========================================+
1132 // 10020 | Frame Pointer |
1133 // 10024 | Frame Pointer |
1134 // +-------------------------------------------+
1135 // 10028 | Link Register |
1136 // 1002c | Link Register |
1137 // +===========================================+
1138 // 10030 | ........ | ........ | ........ | ........ |
1139 // 10034 | ........ | ........ | ........ | ........ |
1140 // +-------------------------------------------+
1141 // 10038 | ........ | ........ | ........ | ........ |
1142 // 1003c | ........ | ........ | ........ | ........ |
1143 // +-------------------------------------------+
1145 // [sp] = 10030 :: >>initial value<<
1146 // sp = 10020 :: stp fp, lr, [sp, #-16]!
1147 // fp = sp == 10020 :: mov fp, sp
1148 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1149 // sp == 10010 :: >>final value<<
1151 // The frame pointer (w29) points to address 10020. If we use an offset of
1152 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1153 // for w27, and -32 for w28:
1156 // .cfi_def_cfa w29, 16
1158 // .cfi_offset w30, -8
1160 // .cfi_offset w29, -16
1162 // .cfi_offset w27, -24
1164 // .cfi_offset w28, -32
1167 // Define the current CFA rule to use the provided FP.
1168 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1169 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
1170 nullptr, Reg, 2 * StackGrowth - FixedObject));
1171 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1172 .addCFIIndex(CFIIndex)
1173 .setMIFlags(MachineInstr::FrameSetup);
1175 // Encode the stack size of the leaf function.
1176 unsigned CFIIndex = MF.addFrameInst(
1177 MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
1178 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1179 .addCFIIndex(CFIIndex)
1180 .setMIFlags(MachineInstr::FrameSetup);
1183 // Now emit the moves for whatever callee saved regs we have (including FP,
1184 // LR if those are saved).
1185 emitCalleeSavedFrameMoves(MBB, MBBI);
1189 static void InsertReturnAddressAuth(MachineFunction &MF,
1190 MachineBasicBlock &MBB) {
1191 if (!ShouldSignReturnAddress(MF))
1193 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1194 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1196 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1198 if (MBBI != MBB.end())
1199 DL = MBBI->getDebugLoc();
1201 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1202 // this instruction can safely used for any v8a architecture.
1203 // From v8.3a onwards there are optimised authenticate LR and return
1204 // instructions, namely RETA{A,B}, that can be used instead.
1205 if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1206 MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1207 BuildMI(MBB, MBBI, DL,
1208 TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1209 .copyImplicitOps(*MBBI);
1214 TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1215 .setMIFlag(MachineInstr::FrameDestroy);
1219 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1220 switch (MI.getOpcode()) {
1223 case AArch64::CATCHRET:
1224 case AArch64::CLEANUPRET:
1229 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
1230 MachineBasicBlock &MBB) const {
1231 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1232 MachineFrameInfo &MFI = MF.getFrameInfo();
1233 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1234 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1236 bool IsTailCallReturn = false;
1237 bool NeedsWinCFI = needsWinCFI(MF);
1238 bool IsFunclet = false;
1240 if (MBB.end() != MBBI) {
1241 DL = MBBI->getDebugLoc();
1242 unsigned RetOpcode = MBBI->getOpcode();
1243 IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
1244 RetOpcode == AArch64::TCRETURNri ||
1245 RetOpcode == AArch64::TCRETURNriBTI;
1246 IsFunclet = isFuncletReturnInstr(*MBBI);
1249 int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
1250 : MFI.getStackSize();
1251 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1253 // All calls are tail calls in GHC calling conv, and functions have no
1254 // prologue/epilogue.
1255 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1258 // Initial and residual are named for consistency with the prologue. Note that
1259 // in the epilogue, the residual adjustment is executed first.
1260 uint64_t ArgumentPopSize = 0;
1261 if (IsTailCallReturn) {
1262 MachineOperand &StackAdjust = MBBI->getOperand(1);
1264 // For a tail-call in a callee-pops-arguments environment, some or all of
1265 // the stack may actually be in use for the call's arguments, this is
1266 // calculated during LowerCall and consumed here...
1267 ArgumentPopSize = StackAdjust.getImm();
1269 // ... otherwise the amount to pop is *all* of the argument space,
1270 // conveniently stored in the MachineFunctionInfo by
1271 // LowerFormalArguments. This will, of course, be zero for the C calling
1273 ArgumentPopSize = AFI->getArgumentStackToRestore();
1276 // The stack frame should be like below,
1278 // ---------------------- ---
1280 // | BytesInStackArgArea| CalleeArgStackSize
1281 // | (NumReusableBytes) | (of tail call)
1284 // ---------------------| --- |
1286 // | CalleeSavedReg | | |
1287 // | (CalleeSavedStackSize)| | |
1289 // ---------------------| | NumBytes
1290 // | | StackSize (StackAdjustUp)
1291 // | LocalStackSize | | |
1292 // | (covering callee | | |
1295 // ---------------------- --- ---
1297 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1298 // = StackSize + ArgumentPopSize
1300 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1301 // it as the 2nd argument of AArch64ISD::TC_RETURN.
1303 auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1306 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1307 // Var args are accounted for in the containing function, so don't
1308 // include them for funclets.
1309 unsigned FixedObject =
1310 (IsWin64 && !IsFunclet) ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1312 uint64_t AfterCSRPopSize = ArgumentPopSize;
1313 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1314 // We cannot rely on the local stack size set in emitPrologue if the function
1315 // has funclets, as funclets have different local stack size requirements, and
1316 // the current value set in emitPrologue may be that of the containing
1318 if (MF.hasEHFunclets())
1319 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1320 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1321 // Assume we can't combine the last pop with the sp restore.
1323 if (!CombineSPBump && PrologueSaveSize != 0) {
1324 MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1325 while (AArch64InstrInfo::isSEHInstruction(*Pop))
1326 Pop = std::prev(Pop);
1327 // Converting the last ldp to a post-index ldp is valid only if the last
1328 // ldp's offset is 0.
1329 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1330 // If the offset is 0, convert it to a post-index ldp.
1331 if (OffsetOp.getImm() == 0)
1332 convertCalleeSaveRestoreToSPPrePostIncDec(
1333 MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, false);
1335 // If not, make sure to emit an add after the last ldp.
1336 // We're doing this by transfering the size to be restored from the
1337 // adjustment *before* the CSR pops to the adjustment *after* the CSR
1339 AfterCSRPopSize += PrologueSaveSize;
1343 // Move past the restores of the callee-saved registers.
1344 // If we plan on combining the sp bump of the local stack size and the callee
1345 // save stack size, we might need to adjust the CSR save and restore offsets.
1346 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
1347 MachineBasicBlock::iterator Begin = MBB.begin();
1348 while (LastPopI != Begin) {
1350 if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
1353 } else if (CombineSPBump)
1354 fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
1359 BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1360 .setMIFlag(MachineInstr::FrameDestroy);
1362 // If there is a single SP update, insert it before the ret and we're done.
1363 if (CombineSPBump) {
1364 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1365 NumBytes + AfterCSRPopSize, TII, MachineInstr::FrameDestroy,
1366 false, NeedsWinCFI);
1368 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1369 TII->get(AArch64::SEH_EpilogEnd))
1370 .setMIFlag(MachineInstr::FrameDestroy);
1374 NumBytes -= PrologueSaveSize;
1375 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1378 bool RedZone = canUseRedZone(MF);
1379 // If this was a redzone leaf function, we don't need to restore the
1380 // stack pointer (but we may need to pop stack args for fastcc).
1381 if (RedZone && AfterCSRPopSize == 0)
1384 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1385 int StackRestoreBytes = RedZone ? 0 : NumBytes;
1386 if (NoCalleeSaveRestore)
1387 StackRestoreBytes += AfterCSRPopSize;
1389 // If we were able to combine the local stack pop with the argument pop,
1391 bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1393 // If we're done after this, make sure to help the load store optimizer.
1395 adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1397 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1398 StackRestoreBytes, TII, MachineInstr::FrameDestroy, false,
1402 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1403 TII->get(AArch64::SEH_EpilogEnd))
1404 .setMIFlag(MachineInstr::FrameDestroy);
1411 // Restore the original stack pointer.
1412 // FIXME: Rather than doing the math here, we should instead just use
1413 // non-post-indexed loads for the restores if we aren't actually going to
1414 // be able to save any instructions.
1415 if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned()))
1416 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1417 -AFI->getCalleeSavedStackSize() + 16, TII,
1418 MachineInstr::FrameDestroy, false, NeedsWinCFI);
1420 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
1421 MachineInstr::FrameDestroy, false, NeedsWinCFI);
1423 // This must be placed after the callee-save restore code because that code
1424 // assumes the SP is at the same location as it was after the callee-save save
1425 // code in the prologue.
1426 if (AfterCSRPopSize) {
1427 // Find an insertion point for the first ldp so that it goes before the
1428 // shadow call stack epilog instruction. This ensures that the restore of
1429 // lr from x18 is placed after the restore from sp.
1430 auto FirstSPPopI = MBB.getFirstTerminator();
1431 while (FirstSPPopI != Begin) {
1432 auto Prev = std::prev(FirstSPPopI);
1433 if (Prev->getOpcode() != AArch64::LDRXpre ||
1434 Prev->getOperand(0).getReg() == AArch64::SP)
1439 adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1441 emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1442 AfterCSRPopSize, TII, MachineInstr::FrameDestroy, false,
1446 BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1447 .setMIFlag(MachineInstr::FrameDestroy);
1450 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1451 /// debug info. It's the same as what we use for resolving the code-gen
1452 /// references for now. FIXME: This can go wrong when references are
1453 /// SP-relative and simple call frames aren't used.
1454 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
1456 unsigned &FrameReg) const {
1457 return resolveFrameIndexReference(MF, FI, FrameReg);
1460 int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
1461 int FI, unsigned &FrameReg,
1462 bool PreferFP) const {
1463 const MachineFrameInfo &MFI = MF.getFrameInfo();
1464 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1465 MF.getSubtarget().getRegisterInfo());
1466 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1467 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1469 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1470 unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1471 int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
1472 int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
1473 bool isFixed = MFI.isFixedObjectIndex(FI);
1474 bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
1475 -((int)AFI->getCalleeSavedStackSize());
1477 // Use frame pointer to reference fixed objects. Use it for locals if
1478 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1479 // reliable as a base). Make sure useFPForScavengingIndex() does the
1480 // right thing for the emergency spill slot.
1482 if (AFI->hasStackFrame()) {
1483 // Note: Keeping the following as multiple 'if' statements rather than
1484 // merging to a single expression for readability.
1486 // Argument access should always use the FP.
1489 } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1490 // References to the CSR area must use FP if we're re-aligning the stack
1491 // since the dynamically-sized alignment padding is between the SP/BP and
1493 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1495 } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1496 // If the FPOffset is negative, we have to keep in mind that the
1497 // available offset range for negative offsets is smaller than for
1498 // positive ones. If an offset is
1499 // available via the FP and the SP, use whichever is closest.
1500 bool FPOffsetFits = FPOffset >= -256;
1501 PreferFP |= Offset > -FPOffset;
1503 if (MFI.hasVarSizedObjects()) {
1504 // If we have variable sized objects, we can use either FP or BP, as the
1505 // SP offset is unknown. We can use the base pointer if we have one and
1506 // FP is not preferred. If not, we're stuck with using FP.
1507 bool CanUseBP = RegInfo->hasBasePointer(MF);
1508 if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1510 else if (!CanUseBP) // Can't use BP. Forced to use FP.
1512 // else we can use BP and FP, but the offset from FP won't fit.
1513 // That will make us scavenge registers which we can probably avoid by
1514 // using BP. If it won't fit for BP either, we'll scavenge anyway.
1515 } else if (FPOffset >= 0) {
1516 // Use SP or FP, whichever gives us the best chance of the offset
1517 // being in range for direct access. If the FPOffset is positive,
1518 // that'll always be best, as the SP will be even further away.
1520 } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1521 // Funclets access the locals contained in the parent's stack frame
1522 // via the frame pointer, so we have to use the FP in the parent
1525 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1526 "Funclets should only be present on Win64");
1529 // We have the choice between FP and (SP or BP).
1530 if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1536 assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1537 "In the presence of dynamic stack pointer realignment, "
1538 "non-argument/CSR objects cannot be accessed through the frame pointer");
1541 FrameReg = RegInfo->getFrameRegister(MF);
1545 // Use the base pointer if we have one.
1546 if (RegInfo->hasBasePointer(MF))
1547 FrameReg = RegInfo->getBaseRegister();
1549 assert(!MFI.hasVarSizedObjects() &&
1550 "Can't use SP when we have var sized objects.");
1551 FrameReg = AArch64::SP;
1552 // If we're using the red zone for this function, the SP won't actually
1553 // be adjusted, so the offsets will be negative. They're also all
1554 // within range of the signed 9-bit immediate instructions.
1555 if (canUseRedZone(MF))
1556 Offset -= AFI->getLocalStackSize();
1562 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1563 // Do not set a kill flag on values that are also marked as live-in. This
1564 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1565 // callee saved registers.
1566 // Omitting the kill flags is conservatively correct even if the live-in
1567 // is not used after all.
1568 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1569 return getKillRegState(!IsLiveIn);
1572 static bool produceCompactUnwindFrame(MachineFunction &MF) {
1573 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1574 AttributeList Attrs = MF.getFunction().getAttributes();
1575 return Subtarget.isTargetMachO() &&
1576 !(Subtarget.getTargetLowering()->supportSwiftError() &&
1577 Attrs.hasAttrSomewhere(Attribute::SwiftError));
1580 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
1582 // If we are generating register pairs for a Windows function that requires
1583 // EH support, then pair consecutive registers only. There are no unwind
1584 // opcodes for saves/restores of non-consectuve register pairs.
1585 // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
1586 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1588 // TODO: LR can be paired with any register. We don't support this yet in
1589 // the MCLayer. We need to add support for the save_lrpair unwind code.
1592 if (Reg2 == Reg1 + 1)
1599 struct RegPairInfo {
1600 unsigned Reg1 = AArch64::NoRegister;
1601 unsigned Reg2 = AArch64::NoRegister;
1604 enum RegType { GPR, FPR64, FPR128 } Type;
1606 RegPairInfo() = default;
1608 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1611 } // end anonymous namespace
1613 static void computeCalleeSaveRegisterPairs(
1614 MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
1615 const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
1616 bool &NeedShadowCallStackProlog) {
1621 bool NeedsWinCFI = needsWinCFI(MF);
1622 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1623 MachineFrameInfo &MFI = MF.getFrameInfo();
1624 CallingConv::ID CC = MF.getFunction().getCallingConv();
1625 unsigned Count = CSI.size();
1627 // MachO's compact unwind format relies on all registers being stored in
1629 assert((!produceCompactUnwindFrame(MF) ||
1630 CC == CallingConv::PreserveMost ||
1631 (Count & 1) == 0) &&
1632 "Odd number of callee-saved regs to spill!");
1633 int Offset = AFI->getCalleeSavedStackSize();
1634 // On Linux, we will have either one or zero non-paired register. On Windows
1635 // with CFI, we can have multiple unpaired registers in order to utilize the
1636 // available unwind codes. This flag assures that the alignment fixup is done
1637 // only once, as intened.
1638 bool FixupDone = false;
1639 for (unsigned i = 0; i < Count; ++i) {
1641 RPI.Reg1 = CSI[i].getReg();
1643 if (AArch64::GPR64RegClass.contains(RPI.Reg1))
1644 RPI.Type = RegPairInfo::GPR;
1645 else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
1646 RPI.Type = RegPairInfo::FPR64;
1647 else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
1648 RPI.Type = RegPairInfo::FPR128;
1650 llvm_unreachable("Unsupported register class.");
1652 // Add the next reg to the pair if it is in the same register class.
1653 if (i + 1 < Count) {
1654 unsigned NextReg = CSI[i + 1].getReg();
1656 case RegPairInfo::GPR:
1657 if (AArch64::GPR64RegClass.contains(NextReg) &&
1658 !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1661 case RegPairInfo::FPR64:
1662 if (AArch64::FPR64RegClass.contains(NextReg) &&
1663 !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1666 case RegPairInfo::FPR128:
1667 if (AArch64::FPR128RegClass.contains(NextReg))
1673 // If either of the registers to be saved is the lr register, it means that
1674 // we also need to save lr in the shadow call stack.
1675 if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
1676 MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
1677 if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
1678 report_fatal_error("Must reserve x18 to use shadow call stack");
1679 NeedShadowCallStackProlog = true;
1682 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
1683 // list to come in sorted by frame index so that we can issue the store
1684 // pair instructions directly. Assert if we see anything otherwise.
1686 // The order of the registers in the list is controlled by
1687 // getCalleeSavedRegs(), so they will always be in-order, as well.
1688 assert((!RPI.isPaired() ||
1689 (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
1690 "Out of order callee saved regs!");
1692 // MachO's compact unwind format relies on all registers being stored in
1693 // adjacent register pairs.
1694 assert((!produceCompactUnwindFrame(MF) ||
1695 CC == CallingConv::PreserveMost ||
1697 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
1698 RPI.Reg1 + 1 == RPI.Reg2))) &&
1699 "Callee-save registers not saved as adjacent register pair!");
1701 RPI.FrameIdx = CSI[i].getFrameIdx();
1703 int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
1704 Offset -= RPI.isPaired() ? 2 * Scale : Scale;
1706 // Round up size of non-pair to pair size if we need to pad the
1707 // callee-save area to ensure 16-byte alignment.
1708 if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
1709 RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
1712 assert(Offset % 16 == 0);
1713 assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
1714 MFI.setObjectAlignment(RPI.FrameIdx, 16);
1717 assert(Offset % Scale == 0);
1718 RPI.Offset = Offset / Scale;
1719 assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
1720 "Offset out of bounds for LDP/STP immediate");
1722 RegPairs.push_back(RPI);
1728 bool AArch64FrameLowering::spillCalleeSavedRegisters(
1729 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1730 const std::vector<CalleeSavedInfo> &CSI,
1731 const TargetRegisterInfo *TRI) const {
1732 MachineFunction &MF = *MBB.getParent();
1733 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1734 bool NeedsWinCFI = needsWinCFI(MF);
1736 SmallVector<RegPairInfo, 8> RegPairs;
1738 bool NeedShadowCallStackProlog = false;
1739 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1740 NeedShadowCallStackProlog);
1741 const MachineRegisterInfo &MRI = MF.getRegInfo();
1743 if (NeedShadowCallStackProlog) {
1744 // Shadow call stack prolog: str x30, [x18], #8
1745 BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
1746 .addReg(AArch64::X18, RegState::Define)
1747 .addReg(AArch64::LR)
1748 .addReg(AArch64::X18)
1750 .setMIFlag(MachineInstr::FrameSetup);
1753 BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
1754 .setMIFlag(MachineInstr::FrameSetup);
1756 if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
1757 // Emit a CFI instruction that causes 8 to be subtracted from the value of
1758 // x18 when unwinding past this frame.
1759 static const char CFIInst[] = {
1760 dwarf::DW_CFA_val_expression,
1763 static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1764 static_cast<char>(-8) & 0x7f, // addend (sleb128)
1767 MF.addFrameInst(MCCFIInstruction::createEscape(nullptr, CFIInst));
1768 BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
1769 .addCFIIndex(CFIIndex)
1770 .setMIFlag(MachineInstr::FrameSetup);
1773 // This instruction also makes x18 live-in to the entry block.
1774 MBB.addLiveIn(AArch64::X18);
1777 for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
1779 RegPairInfo RPI = *RPII;
1780 unsigned Reg1 = RPI.Reg1;
1781 unsigned Reg2 = RPI.Reg2;
1784 // Issue sequence of spills for cs regs. The first spill may be converted
1785 // to a pre-decrement store later by emitPrologue if the callee-save stack
1786 // area allocation can't be combined with the local stack area allocation.
1788 // stp x22, x21, [sp, #0] // addImm(+0)
1789 // stp x20, x19, [sp, #16] // addImm(+2)
1790 // stp fp, lr, [sp, #32] // addImm(+4)
1791 // Rationale: This sequence saves uop updates compared to a sequence of
1792 // pre-increment spills like stp xi,xj,[sp,#-16]!
1793 // Note: Similar rationale and sequence for restores in epilog.
1794 unsigned Size, Align;
1796 case RegPairInfo::GPR:
1797 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
1801 case RegPairInfo::FPR64:
1802 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
1806 case RegPairInfo::FPR128:
1807 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
1812 LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
1813 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1814 dbgs() << ") -> fi#(" << RPI.FrameIdx;
1815 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1818 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
1819 "Windows unwdinding requires a consecutive (FP,LR) pair");
1820 // Windows unwind codes require consecutive registers if registers are
1821 // paired. Make the switch here, so that the code below will save (x,x+1)
1823 unsigned FrameIdxReg1 = RPI.FrameIdx;
1824 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
1825 if (NeedsWinCFI && RPI.isPaired()) {
1826 std::swap(Reg1, Reg2);
1827 std::swap(FrameIdxReg1, FrameIdxReg2);
1829 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
1830 if (!MRI.isReserved(Reg1))
1831 MBB.addLiveIn(Reg1);
1832 if (RPI.isPaired()) {
1833 if (!MRI.isReserved(Reg2))
1834 MBB.addLiveIn(Reg2);
1835 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
1836 MIB.addMemOperand(MF.getMachineMemOperand(
1837 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
1838 MachineMemOperand::MOStore, Size, Align));
1840 MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
1841 .addReg(AArch64::SP)
1842 .addImm(RPI.Offset) // [sp, #offset*scale],
1843 // where factor*scale is implicit
1844 .setMIFlag(MachineInstr::FrameSetup);
1845 MIB.addMemOperand(MF.getMachineMemOperand(
1846 MachinePointerInfo::getFixedStack(MF,FrameIdxReg1),
1847 MachineMemOperand::MOStore, Size, Align));
1849 InsertSEH(MIB, TII, MachineInstr::FrameSetup);
1855 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
1856 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1857 std::vector<CalleeSavedInfo> &CSI,
1858 const TargetRegisterInfo *TRI) const {
1859 MachineFunction &MF = *MBB.getParent();
1860 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1862 SmallVector<RegPairInfo, 8> RegPairs;
1863 bool NeedsWinCFI = needsWinCFI(MF);
1865 if (MI != MBB.end())
1866 DL = MI->getDebugLoc();
1868 bool NeedShadowCallStackProlog = false;
1869 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1870 NeedShadowCallStackProlog);
1872 auto EmitMI = [&](const RegPairInfo &RPI) {
1873 unsigned Reg1 = RPI.Reg1;
1874 unsigned Reg2 = RPI.Reg2;
1876 // Issue sequence of restores for cs regs. The last restore may be converted
1877 // to a post-increment load later by emitEpilogue if the callee-save stack
1878 // area allocation can't be combined with the local stack area allocation.
1880 // ldp fp, lr, [sp, #32] // addImm(+4)
1881 // ldp x20, x19, [sp, #16] // addImm(+2)
1882 // ldp x22, x21, [sp, #0] // addImm(+0)
1883 // Note: see comment in spillCalleeSavedRegisters()
1885 unsigned Size, Align;
1887 case RegPairInfo::GPR:
1888 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
1892 case RegPairInfo::FPR64:
1893 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
1897 case RegPairInfo::FPR128:
1898 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
1903 LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
1904 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1905 dbgs() << ") -> fi#(" << RPI.FrameIdx;
1906 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1909 // Windows unwind codes require consecutive registers if registers are
1910 // paired. Make the switch here, so that the code below will save (x,x+1)
1912 unsigned FrameIdxReg1 = RPI.FrameIdx;
1913 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
1914 if (NeedsWinCFI && RPI.isPaired()) {
1915 std::swap(Reg1, Reg2);
1916 std::swap(FrameIdxReg1, FrameIdxReg2);
1918 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
1919 if (RPI.isPaired()) {
1920 MIB.addReg(Reg2, getDefRegState(true));
1921 MIB.addMemOperand(MF.getMachineMemOperand(
1922 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
1923 MachineMemOperand::MOLoad, Size, Align));
1925 MIB.addReg(Reg1, getDefRegState(true))
1926 .addReg(AArch64::SP)
1927 .addImm(RPI.Offset) // [sp, #offset*scale]
1928 // where factor*scale is implicit
1929 .setMIFlag(MachineInstr::FrameDestroy);
1930 MIB.addMemOperand(MF.getMachineMemOperand(
1931 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
1932 MachineMemOperand::MOLoad, Size, Align));
1934 InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
1936 if (ReverseCSRRestoreSeq)
1937 for (const RegPairInfo &RPI : reverse(RegPairs))
1940 for (const RegPairInfo &RPI : RegPairs)
1943 if (NeedShadowCallStackProlog) {
1944 // Shadow call stack epilog: ldr x30, [x18, #-8]!
1945 BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
1946 .addReg(AArch64::X18, RegState::Define)
1947 .addReg(AArch64::LR, RegState::Define)
1948 .addReg(AArch64::X18)
1950 .setMIFlag(MachineInstr::FrameDestroy);
1956 void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
1957 BitVector &SavedRegs,
1958 RegScavenger *RS) const {
1959 // All calls are tail calls in GHC calling conv, and functions have no
1960 // prologue/epilogue.
1961 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1964 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1965 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1966 MF.getSubtarget().getRegisterInfo());
1967 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1968 unsigned UnspilledCSGPR = AArch64::NoRegister;
1969 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
1971 MachineFrameInfo &MFI = MF.getFrameInfo();
1972 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
1974 unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
1975 ? RegInfo->getBaseRegister()
1976 : (unsigned)AArch64::NoRegister;
1978 unsigned ExtraCSSpill = 0;
1979 // Figure out which callee-saved registers to save/restore.
1980 for (unsigned i = 0; CSRegs[i]; ++i) {
1981 const unsigned Reg = CSRegs[i];
1983 // Add the base pointer register to SavedRegs if it is callee-save.
1984 if (Reg == BasePointerReg)
1987 bool RegUsed = SavedRegs.test(Reg);
1988 unsigned PairedReg = CSRegs[i ^ 1];
1990 if (AArch64::GPR64RegClass.contains(Reg) &&
1991 !RegInfo->isReservedReg(MF, Reg)) {
1992 UnspilledCSGPR = Reg;
1993 UnspilledCSGPRPaired = PairedReg;
1998 // MachO's compact unwind format relies on all registers being stored in
2000 // FIXME: the usual format is actually better if unwinding isn't needed.
2001 if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2002 !SavedRegs.test(PairedReg)) {
2003 SavedRegs.set(PairedReg);
2004 if (AArch64::GPR64RegClass.contains(PairedReg) &&
2005 !RegInfo->isReservedReg(MF, PairedReg))
2006 ExtraCSSpill = PairedReg;
2010 // Calculates the callee saved stack size.
2011 unsigned CSStackSize = 0;
2012 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2013 const MachineRegisterInfo &MRI = MF.getRegInfo();
2014 for (unsigned Reg : SavedRegs.set_bits())
2015 CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
2017 // Save number of saved regs, so we can easily update CSStackSize later.
2018 unsigned NumSavedRegs = SavedRegs.count();
2020 // The frame record needs to be created by saving the appropriate registers
2021 unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
2023 windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2024 SavedRegs.set(AArch64::FP);
2025 SavedRegs.set(AArch64::LR);
2028 LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
2030 : SavedRegs.set_bits()) dbgs()
2031 << ' ' << printReg(Reg, RegInfo);
2034 // If any callee-saved registers are used, the frame cannot be eliminated.
2035 bool CanEliminateFrame = SavedRegs.count() == 0;
2037 // The CSR spill slots have not been allocated yet, so estimateStackSize
2038 // won't include them.
2039 unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2040 bool BigStack = (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2041 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2042 AFI->setHasStackFrame(true);
2044 // Estimate if we might need to scavenge a register at some point in order
2045 // to materialize a stack offset. If so, either spill one additional
2046 // callee-saved register or reserve a special spill slot to facilitate
2047 // register scavenging. If we already spilled an extra callee-saved register
2048 // above to keep the number of spills even, we don't need to do anything else
2051 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2052 LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2053 << " to get a scratch register.\n");
2054 SavedRegs.set(UnspilledCSGPR);
2055 // MachO's compact unwind format relies on all registers being stored in
2056 // pairs, so if we need to spill one extra for BigStack, then we need to
2058 if (produceCompactUnwindFrame(MF))
2059 SavedRegs.set(UnspilledCSGPRPaired);
2060 ExtraCSSpill = UnspilledCSGPRPaired;
2063 // If we didn't find an extra callee-saved register to spill, create
2064 // an emergency spill slot.
2065 if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2066 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2067 const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2068 unsigned Size = TRI->getSpillSize(RC);
2069 unsigned Align = TRI->getSpillAlignment(RC);
2070 int FI = MFI.CreateStackObject(Size, Align, false);
2071 RS->addScavengingFrameIndex(FI);
2072 LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2073 << " as the emergency spill slot.\n");
2077 // Adding the size of additional 64bit GPR saves.
2078 CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2079 unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
2080 LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2081 << EstimatedStackSize + AlignedCSStackSize
2084 // Round up to register pair alignment to avoid additional SP adjustment
2086 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2087 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2090 bool AArch64FrameLowering::enableStackSlotScavenging(
2091 const MachineFunction &MF) const {
2092 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2093 return AFI->hasCalleeSaveStackFreeSpace();
2096 void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
2097 MachineFunction &MF, RegScavenger *RS) const {
2098 // If this function isn't doing Win64-style C++ EH, we don't need to do
2100 if (!MF.hasEHFunclets())
2102 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2103 MachineFrameInfo &MFI = MF.getFrameInfo();
2104 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2106 MachineBasicBlock &MBB = MF.front();
2107 auto MBBI = MBB.begin();
2108 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2111 if (MBBI->isTerminator())
2114 // Create an UnwindHelp object.
2116 MFI.CreateStackObject(/*size*/8, /*alignment*/16, false);
2117 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2118 // We need to store -2 into the UnwindHelp object at the start of the
2121 RS->enterBasicBlock(MBB);
2122 unsigned DstReg = RS->scavengeRegister(&AArch64::GPR64RegClass, MBBI, 0);
2123 BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2124 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2125 .addReg(DstReg, getKillRegState(true))
2126 .addFrameIndex(UnwindHelpFI)
2130 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP before
2131 /// the update. This is easily retrieved as it is exactly the offset that is set
2132 /// in processFunctionBeforeFrameFinalized.
2133 int AArch64FrameLowering::getFrameIndexReferencePreferSP(
2134 const MachineFunction &MF, int FI, unsigned &FrameReg,
2135 bool IgnoreSPUpdates) const {
2136 const MachineFrameInfo &MFI = MF.getFrameInfo();
2137 LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
2138 << MFI.getObjectOffset(FI) << "\n");
2139 FrameReg = AArch64::SP;
2140 return MFI.getObjectOffset(FI);
2143 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
2144 /// the parent's frame pointer
2145 unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
2146 const MachineFunction &MF) const {
2150 /// Funclets only need to account for space for the callee saved registers,
2151 /// as the locals are accounted for in the parent's stack frame.
2152 unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
2153 const MachineFunction &MF) const {
2154 // This is the size of the pushed CSRs.
2156 MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
2157 // This is the amount of stack a funclet needs to allocate.
2158 return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
2159 getStackAlignment());